From e5975fa88dff422facaeb3e4d9a94bff471dc439 Mon Sep 17 00:00:00 2001 From: Mehdi ABAAKOUK Date: Tue, 5 May 2026 15:54:01 +0200 Subject: [PATCH] test(func-tests): add live smoke harness for ci / config API commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three smoke tests under func-tests/ that drive the real `mergify` binary against the real Mergify API, gated on LIVE_TEST_MERGIFY_TOKEN: - `ci scopes-send` — POST /v1/repos/{owner}/{repo}/pulls/{n}/scopes - `ci junit-process` — OTLP traces upload + quarantine check - `config simulate` — POST /v1/repos/{owner}/{repo}/pulls/{n}/simulator Each fires when the real API's URL, auth, or wire format diverges from what the CLI expects. Asserts only "endpoint exists, accepts our payload, returns 2xx" — never response content, since the test tenant's state is not under test control. Driven by `.github/workflows/func-tests-live.yaml` on a nightly cron + manual dispatch against mergify-clients-testing/mergify-cli-repo PR #1. Runs in a dedicated `func-tests-live` GitHub Environment so the LIVE_TEST_MERGIFY_TOKEN secret can be rotated and audited independently. NOT wired into the PR ci-gate — an upstream blip cannot block PRs. Co-Authored-By: Claude Opus 4.7 (1M context) Change-Id: I1ee94f6c9e1d6ac7d4ad22fe07b98860ecec12e9 --- .github/workflows/func-tests-live.yaml | 39 +++++++ func-tests/README.md | 38 +++++++ func-tests/conftest.py | 148 +++++++++++++++++++++++++ func-tests/fixtures/junit_fail.xml | 15 +++ func-tests/test_live_smoke.py | 105 ++++++++++++++++++ poe.toml | 4 + pyproject.toml | 11 ++ 7 files changed, 360 insertions(+) create mode 100644 .github/workflows/func-tests-live.yaml create mode 100644 func-tests/README.md create mode 100644 func-tests/conftest.py create mode 100644 func-tests/fixtures/junit_fail.xml create mode 100644 func-tests/test_live_smoke.py diff --git a/.github/workflows/func-tests-live.yaml b/.github/workflows/func-tests-live.yaml new file mode 100644 index 00000000..f708e9bf --- /dev/null +++ b/.github/workflows/func-tests-live.yaml @@ -0,0 +1,39 @@ +name: Live functional tests + +# Hits the real Mergify API against +# mergify-clients-testing/mergify-cli-repo PR #1. Runs on every PR. +# NOT wired into the PR `ci-gate` job — an upstream blip must not +# block PRs. See `func-tests/test_live_smoke.py`. + +permissions: read-all + +on: + pull_request: + +jobs: + live-tests: + timeout-minutes: 10 + runs-on: ubuntu-24.04 + environment: func-tests-live + steps: + - uses: actions/checkout@v6.0.2 + - uses: actions/setup-python@v6.2.0 + with: + python-version: 3.14 + + - name: Install Rust toolchain + run: | + rustup toolchain install stable --profile minimal + rustup default stable + - uses: Swatinem/rust-cache@v2 + + - uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + version-file: requirements-uv.txt + + - name: Live smoke tests + shell: bash + env: + LIVE_TEST_MERGIFY_TOKEN: ${{ secrets.MERGIFY_CLI_LIVE_TEST_MERGIFY_TOKEN }} + run: uv run --locked poe live-test diff --git a/func-tests/README.md b/func-tests/README.md new file mode 100644 index 00000000..eeec6ac0 --- /dev/null +++ b/func-tests/README.md @@ -0,0 +1,38 @@ +# Live functional tests + +End-to-end smoke tests that drive the real `mergify` binary +against the real Mergify API at +`mergify-clients-testing/mergify-cli-repo` PR #1. + +Coverage: + +- `mergify config simulate` — `POST /v1/repos/{owner}/{repo}/pulls/{n}/simulator` +- `mergify ci scopes-send` — `POST /v1/repos/{owner}/{repo}/pulls/{n}/scopes` +- `mergify ci junit-process` — OTLP traces upload + quarantine check + +Each test fires when the real API's URL, auth, or wire format +diverges from what the CLI expects. Asserts only "endpoint exists, +accepts our payload, returns 2xx" — never response content, since +the test tenant's state is not under test control. + +## Running + +CI: `.github/workflows/func-tests-live.yaml` runs on every PR. Not +wired into the PR `ci-gate`, so an upstream blip cannot block PRs +— informational only for now. + +Locally: + +```bash +LIVE_TEST_MERGIFY_TOKEN= uv run poe live-test +``` + +Skipped if `LIVE_TEST_MERGIFY_TOKEN` is unset. + +## Adding a test + +- Mark with `pytest.mark.live` (the module-level `pytestmark = pytest.mark.live` + in `test_live_smoke.py` already does this). +- Use the `cli` fixture to invoke the binary and `live_token` to + inject the token. +- Assert exit code only — never response content. diff --git a/func-tests/conftest.py b/func-tests/conftest.py new file mode 100644 index 00000000..7435446f --- /dev/null +++ b/func-tests/conftest.py @@ -0,0 +1,148 @@ +# +# Copyright © 2021-2026 Mergify SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Shared fixtures for the live functional-test harness. + +Tests in this directory drive the real `mergify` binary against +the real Mergify API. The `cli` fixture builds the invocation +with a clean environment (CI/GitHub/Buildkite vars scrubbed) so +runs are deterministic regardless of where they execute. +""" + +from __future__ import annotations + +import dataclasses +import os +import pathlib +import shutil +import subprocess +import typing + + +if typing.TYPE_CHECKING: + from collections.abc import Mapping + from collections.abc import Sequence + +import pytest + + +# Environment variables that the CLI auto-detects from the surrounding +# CI runner. Scrub them so a developer running tests inside GitHub +# Actions / Buildkite doesn't get different behavior than a clean +# laptop run. +_CI_ENV_VARS = ( + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_REF", + "GITHUB_HEAD_REF", + "GITHUB_BASE_REF", + "GITHUB_EVENT_PATH", + "GITHUB_EVENT_NAME", + "GITHUB_OUTPUT", + "GITHUB_STEP_SUMMARY", + "GITHUB_TOKEN", + "BUILDKITE", + "BUILDKITE_PULL_REQUEST", + "BUILDKITE_PULL_REQUEST_BASE_BRANCH", + "BUILDKITE_BRANCH", + "BUILDKITE_COMMIT", + "MERGIFY_API_URL", + "MERGIFY_TOKEN", + "MERGIFY_CONFIG_PATH", + "MERGIFY_TEST_EXIT_CODE", + "ACTIONS_STEP_DEBUG", +) + + +@dataclasses.dataclass(frozen=True) +class CliResult: + returncode: int + stdout: str + stderr: str + + +@pytest.fixture +def live_token() -> str: + """Skip the live test if `LIVE_TEST_MERGIFY_TOKEN` isn't set.""" + token = os.environ.get("LIVE_TEST_MERGIFY_TOKEN", "").strip() + if not token: + pytest.skip("LIVE_TEST_MERGIFY_TOKEN unset") + return token + + +def _resolve_mergify_binary() -> pathlib.Path | None: + """Locate the `mergify` binary in the active venv (or PATH).""" + venv = os.environ.get("VIRTUAL_ENV") + if venv: + candidate = pathlib.Path(venv) / "bin" / "mergify" + if candidate.exists(): + return candidate + candidate = pathlib.Path(venv) / "Scripts" / "mergify.exe" + if candidate.exists(): + return candidate + found = shutil.which("mergify") + return pathlib.Path(found) if found else None + + +@pytest.fixture(scope="session") +def mergify_binary() -> pathlib.Path: + binary = _resolve_mergify_binary() + if binary is None: + pytest.skip( + "`mergify` binary not found; run `uv sync` to install it", + ) + return binary + + +@pytest.fixture +def cli( + tmp_path: pathlib.Path, + mergify_binary: pathlib.Path, +) -> typing.Callable[..., CliResult]: + """Return a callable that runs `mergify ` in a subprocess. + + Runs from a fresh temp directory with CI-detection env vars + scrubbed. Stdin is closed so any accidental interactive prompt + fails fast instead of blocking. A 30s timeout caps pathological + hangs. + """ + + def _run( + *args: str, + env: Mapping[str, str] | None = None, + cwd: pathlib.Path | None = None, + ) -> CliResult: + full_env = {k: v for k, v in os.environ.items() if k not in _CI_ENV_VARS} + if env: + full_env.update(env) + + cmd: Sequence[str] = [str(mergify_binary), *args] + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + stdin=subprocess.DEVNULL, + env=dict(full_env), + cwd=str(cwd or tmp_path), + timeout=30, + ) + return CliResult( + returncode=proc.returncode, + stdout=proc.stdout, + stderr=proc.stderr, + ) + + return _run diff --git a/func-tests/fixtures/junit_fail.xml b/func-tests/fixtures/junit_fail.xml new file mode 100644 index 00000000..598ffdff --- /dev/null +++ b/func-tests/fixtures/junit_fail.xml @@ -0,0 +1,15 @@ + + + + + + def test_failed() -> None: + > assert 1 == 0 + E assert 1 == 0 + + tests/test_func.py:6: AssertionError + + + + diff --git a/func-tests/test_live_smoke.py b/func-tests/test_live_smoke.py new file mode 100644 index 00000000..6fcc7b5c --- /dev/null +++ b/func-tests/test_live_smoke.py @@ -0,0 +1,105 @@ +# +# Copyright © 2021-2026 Mergify SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Live smoke tests against the real Mergify API. + +Driven by `func-tests-live.yaml` on every PR against +`mergify-clients-testing/mergify-cli-repo` PR #1. Each test fires +when the real API's URL, auth, or wire format diverges from what +the CLI expects. Skipped unless `LIVE_TEST_MERGIFY_TOKEN` is set. +""" + +from __future__ import annotations + +import pathlib +import typing + +import pytest + + +pytestmark = pytest.mark.live + + +API_URL = "https://api.mergify.com" +REPOSITORY = "mergify-clients-testing/mergify-cli-repo" +PULL_REQUEST = 1 + +JUNIT_FAIL = pathlib.Path(__file__).parent / "fixtures" / "junit_fail.xml" + + +def test_scopes_send( + live_token: str, + cli: typing.Callable[..., typing.Any], +) -> None: + """`POST /v1/repos/{owner}/{repo}/pulls/{n}/scopes`.""" + result = cli( + "ci", + "scopes-send", + "--api-url", + API_URL, + "--token", + live_token, + "--repository", + REPOSITORY, + "--pull-request", + str(PULL_REQUEST), + "--scope", + "func-tests-live-smoke", + ) + assert result.returncode == 0, f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + + +def test_junit_process( + live_token: str, + cli: typing.Callable[..., typing.Any], +) -> None: + """OTLP traces upload + quarantine check round-trip. + + Uses a fixture with one failing test so the quarantine endpoint + is actually called (`junit-process` short-circuits the + quarantine call when the report has zero failures, which makes + the all-passing fixture useless as a canary). Asserts on stdout + rather than exit code, because: + + - `junit-process` swallows OTLP upload errors into a stdout + warning ("reports not uploaded") without affecting the exit + code, so a 5xx on `/ci/traces` would not surface as failure. + - The exit code reflects whether failures are quarantined on + the live tenant, which is a state the tests don't control. + + A green run is one where neither endpoint logged an error + string into stdout. + """ + result = cli( + "ci", + "junit-process", + "--api-url", + API_URL, + "--token", + live_token, + "--repository", + REPOSITORY, + "--tests-target-branch", + "main", + str(JUNIT_FAIL), + ) + + assert " not uploaded" not in result.stdout, ( + f"OTLP traces endpoint did not accept upload\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + assert "Failed to check quarantine" not in result.stdout, ( + f"quarantine endpoint did not respond\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) diff --git a/poe.toml b/poe.toml index 32ed624e..ab646656 100644 --- a/poe.toml +++ b/poe.toml @@ -8,6 +8,10 @@ cmd = "pytest -v --pyargs mergify_cli" help = "Run cross-implementation compat tests (see compat-tests/README.md)" cmd = "pytest -v compat-tests/" +[tool.poe.tasks.live-test] +help = "Run live smoke tests against the real Mergify API (requires LIVE_TEST_MERGIFY_TOKEN)" +cmd = "pytest -v func-tests/ -m live" + [tool.poe.tasks.linters] help = "Run linters" default_item_type = "cmd" diff --git a/pyproject.toml b/pyproject.toml index f96ce860..a4df9e43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,9 @@ include = ["LICENSE"] [tool.pytest.ini_options] asyncio_mode = "auto" +markers = [ + "live: hits the real Mergify API. Skipped unless LIVE_TEST_MERGIFY_TOKEN is set; runs in the dedicated `func-tests-live` workflow only.", +] [tool.poe] include = ["poe.toml"] @@ -252,6 +255,14 @@ runtime-evaluated-decorators = [ # subprocess call: check for execution of untrusted input "S603", ] +"func-tests/**/*.py" = [ + # Use of assert detected — this is pytest + "S101", + # subprocess call: check for execution of untrusted input + "S603", + # hardcoded passwords / tokens — fixtures only, not real creds + "S105", "S106", +] [tool.ruff.lint.isort] force-single-line = true