test(func-tests): add functional + live smoke harness for ci API commands

.github/workflows/func-tests-live.yaml

@@ -0,0 +1,39 @@
+name: Live functional tests
+
+# Hits the real Mergify API against
+# mergify-clients-testing/mergify-cli-repo PR #1. Runs on every PR.
+# NOT wired into the PR `ci-gate` job — an upstream blip must not
+# block PRs. See `func-tests/test_live_smoke.py`.
+
+permissions: read-all
+
+on:
+  pull_request:
+
+jobs:
+  live-tests:
+    timeout-minutes: 10
+    runs-on: ubuntu-24.04
+    environment: func-tests-live
+    steps:
+      - uses: actions/checkout@v6.0.2
+      - uses: actions/setup-python@v6.2.0
+        with:
+          python-version: 3.14
+
+      - name: Install Rust toolchain
+        run: |
+          rustup toolchain install stable --profile minimal
+          rustup default stable
+      - uses: Swatinem/rust-cache@v2
+
+      - uses: astral-sh/setup-uv@v8.1.0
+        with:
+          enable-cache: true
+          version-file: requirements-uv.txt
+
+      - name: Live smoke tests
+        shell: bash
+        env:
+          LIVE_TEST_MERGIFY_TOKEN: ${{ secrets.MERGIFY_CLI_LIVE_TEST_MERGIFY_TOKEN }}
+        run: uv run --locked poe live-test

func-tests/README.md

@@ -0,0 +1,38 @@
+# Live functional tests
+
+End-to-end smoke tests that drive the real `mergify` binary
+against the real Mergify API at
+`mergify-clients-testing/mergify-cli-repo` PR #1.
+
+Coverage:
+
+- `mergify config simulate` — `POST /v1/repos/{owner}/{repo}/pulls/{n}/simulator`
+- `mergify ci scopes-send` — `POST /v1/repos/{owner}/{repo}/pulls/{n}/scopes`
+- `mergify ci junit-process` — OTLP traces upload + quarantine check
+
+Each test fires when the real API's URL, auth, or wire format
+diverges from what the CLI expects. Asserts only "endpoint exists,
+accepts our payload, returns 2xx" — never response content, since
+the test tenant's state is not under test control.
+
+## Running
+
+CI: `.github/workflows/func-tests-live.yaml` runs on every PR. Not
+wired into the PR `ci-gate`, so an upstream blip cannot block PRs
+— informational only for now.
+
+Locally:
+
+```bash
+LIVE_TEST_MERGIFY_TOKEN=<app-key> uv run poe live-test
+```
+
+Skipped if `LIVE_TEST_MERGIFY_TOKEN` is unset.
+
+## Adding a test
+
+- Mark with `pytest.mark.live` (the module-level `pytestmark = pytest.mark.live`
+  in `test_live_smoke.py` already does this).
+- Use the `cli` fixture to invoke the binary and `live_token` to
+  inject the token.
+- Assert exit code only — never response content.

func-tests/conftest.py

@@ -0,0 +1,148 @@
+#
+#  Copyright © 2021-2026 Mergify SAS
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Shared fixtures for the live functional-test harness.
+
+Tests in this directory drive the real `mergify` binary against
+the real Mergify API. The `cli` fixture builds the invocation
+with a clean environment (CI/GitHub/Buildkite vars scrubbed) so
+runs are deterministic regardless of where they execute.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+import pathlib
+import shutil
+import subprocess
+import typing
+
+
+if typing.TYPE_CHECKING:
+    from collections.abc import Mapping
+    from collections.abc import Sequence
+
+import pytest
+
+
+# Environment variables that the CLI auto-detects from the surrounding
+# CI runner. Scrub them so a developer running tests inside GitHub
+# Actions / Buildkite doesn't get different behavior than a clean
+# laptop run.
+_CI_ENV_VARS = (
+    "CI",
+    "GITHUB_ACTIONS",
+    "GITHUB_REPOSITORY",
+    "GITHUB_REF",
+    "GITHUB_HEAD_REF",
+    "GITHUB_BASE_REF",
+    "GITHUB_EVENT_PATH",
+    "GITHUB_EVENT_NAME",
+    "GITHUB_OUTPUT",
+    "GITHUB_STEP_SUMMARY",
+    "GITHUB_TOKEN",
+    "BUILDKITE",
+    "BUILDKITE_PULL_REQUEST",
+    "BUILDKITE_PULL_REQUEST_BASE_BRANCH",
+    "BUILDKITE_BRANCH",
+    "BUILDKITE_COMMIT",
+    "MERGIFY_API_URL",
+    "MERGIFY_TOKEN",
+    "MERGIFY_CONFIG_PATH",
+    "MERGIFY_TEST_EXIT_CODE",
+    "ACTIONS_STEP_DEBUG",
+)
+
+
+@dataclasses.dataclass(frozen=True)
+class CliResult:
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+@pytest.fixture
+def live_token() -> str:
+    """Skip the live test if `LIVE_TEST_MERGIFY_TOKEN` isn't set."""
+    token = os.environ.get("LIVE_TEST_MERGIFY_TOKEN", "").strip()
+    if not token:
+        pytest.skip("LIVE_TEST_MERGIFY_TOKEN unset")
+    return token
+
+
+def _resolve_mergify_binary() -> pathlib.Path | None:
+    """Locate the `mergify` binary in the active venv (or PATH)."""
+    venv = os.environ.get("VIRTUAL_ENV")
+    if venv:
+        candidate = pathlib.Path(venv) / "bin" / "mergify"
+        if candidate.exists():
+            return candidate
+        candidate = pathlib.Path(venv) / "Scripts" / "mergify.exe"
+        if candidate.exists():
+            return candidate
+    found = shutil.which("mergify")
+    return pathlib.Path(found) if found else None
+
+
+@pytest.fixture(scope="session")
+def mergify_binary() -> pathlib.Path:
+    binary = _resolve_mergify_binary()
+    if binary is None:
+        pytest.skip(
+            "`mergify` binary not found; run `uv sync` to install it",
+        )
+    return binary
+
+
+@pytest.fixture
+def cli(
+    tmp_path: pathlib.Path,
+    mergify_binary: pathlib.Path,
+) -> typing.Callable[..., CliResult]:
+    """Return a callable that runs `mergify <args>` in a subprocess.
+
+    Runs from a fresh temp directory with CI-detection env vars
+    scrubbed. Stdin is closed so any accidental interactive prompt
+    fails fast instead of blocking. A 30s timeout caps pathological
+    hangs.
+    """
+
+    def _run(
+        *args: str,
+        env: Mapping[str, str] | None = None,
+        cwd: pathlib.Path | None = None,
+    ) -> CliResult:
+        full_env = {k: v for k, v in os.environ.items() if k not in _CI_ENV_VARS}
+        if env:
+            full_env.update(env)
+
+        cmd: Sequence[str] = [str(mergify_binary), *args]
+        proc = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            check=False,
+            stdin=subprocess.DEVNULL,
+            env=dict(full_env),
+            cwd=str(cwd or tmp_path),
+            timeout=30,
+        )
+        return CliResult(
+            returncode=proc.returncode,
+            stdout=proc.stdout,
+            stderr=proc.stderr,
+        )
+
+    return _run

func-tests/fixtures/junit_fail.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+    <testsuite name="pytest" errors="0" failures="1" skipped="0" tests="2" time="0.026"
+               timestamp="2024-08-14T12:25:18.210796+02:00" hostname="func-tests">
+        <testcase classname="tests.test_func" name="test_success" time="0.000"/>
+        <testcase classname="tests.test_func" name="test_failed" time="0.000">
+            <failure message="assert 1 == 0">def test_failed() -&gt; None:
+                &gt; assert 1 == 0
+                E assert 1 == 0
+
+                tests/test_func.py:6: AssertionError
+            </failure>
+        </testcase>
+    </testsuite>
+</testsuites>

func-tests/test_live_smoke.py

@@ -0,0 +1,105 @@
+#
+#  Copyright © 2021-2026 Mergify SAS
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Live smoke tests against the real Mergify API.
+
+Driven by `func-tests-live.yaml` on every PR against
+`mergify-clients-testing/mergify-cli-repo` PR #1. Each test fires
+when the real API's URL, auth, or wire format diverges from what
+the CLI expects. Skipped unless `LIVE_TEST_MERGIFY_TOKEN` is set.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import typing
+
+import pytest
+
+
+pytestmark = pytest.mark.live
+
+
+API_URL = "https://api.mergify.com"
+REPOSITORY = "mergify-clients-testing/mergify-cli-repo"
+PULL_REQUEST = 1
+
+JUNIT_FAIL = pathlib.Path(__file__).parent / "fixtures" / "junit_fail.xml"
+
+
+def test_scopes_send(
+    live_token: str,
+    cli: typing.Callable[..., typing.Any],
+) -> None:
+    """`POST /v1/repos/{owner}/{repo}/pulls/{n}/scopes`."""
+    result = cli(
+        "ci",
+        "scopes-send",
+        "--api-url",
+        API_URL,
+        "--token",
+        live_token,
+        "--repository",
+        REPOSITORY,
+        "--pull-request",
+        str(PULL_REQUEST),
+        "--scope",
+        "func-tests-live-smoke",
+    )
+    assert result.returncode == 0, f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+
+
+def test_junit_process(
+    live_token: str,
+    cli: typing.Callable[..., typing.Any],
+) -> None:
+    """OTLP traces upload + quarantine check round-trip.
+
+    Uses a fixture with one failing test so the quarantine endpoint
+    is actually called (`junit-process` short-circuits the
+    quarantine call when the report has zero failures, which makes
+    the all-passing fixture useless as a canary). Asserts on stdout
+    rather than exit code, because:
+
+    - `junit-process` swallows OTLP upload errors into a stdout
+      warning ("reports not uploaded") without affecting the exit
+      code, so a 5xx on `/ci/traces` would not surface as failure.
+    - The exit code reflects whether failures are quarantined on
+      the live tenant, which is a state the tests don't control.
+
+    A green run is one where neither endpoint logged an error
+    string into stdout.
+    """
+    result = cli(
+        "ci",
+        "junit-process",
+        "--api-url",
+        API_URL,
+        "--token",
+        live_token,
+        "--repository",
+        REPOSITORY,
+        "--tests-target-branch",
+        "main",
+        str(JUNIT_FAIL),
+    )
+
+    assert " not uploaded" not in result.stdout, (
+        f"OTLP traces endpoint did not accept upload\n"
+        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+    )
+    assert "Failed to check quarantine" not in result.stdout, (
+        f"quarantine endpoint did not respond\n"
+        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+    )

poe.toml

@@ -8,6 +8,10 @@ cmd = "pytest -v --pyargs mergify_cli"
 help = "Run cross-implementation compat tests (see compat-tests/README.md)"
 cmd = "pytest -v compat-tests/"
 
+[tool.poe.tasks.live-test]
+help = "Run live smoke tests against the real Mergify API (requires LIVE_TEST_MERGIFY_TOKEN)"
+cmd = "pytest -v func-tests/ -m live"
+
 [tool.poe.tasks.linters]
 help = "Run linters"
 default_item_type = "cmd"

pyproject.toml

@@ -91,6 +91,9 @@ include = ["LICENSE"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+markers = [
+    "live: hits the real Mergify API. Skipped unless LIVE_TEST_MERGIFY_TOKEN is set; runs in the dedicated `func-tests-live` workflow only.",
+]
 
 [tool.poe]
 include = ["poe.toml"]
@@ -252,6 +255,14 @@ runtime-evaluated-decorators = [
     # subprocess call: check for execution of untrusted input
     "S603",
 ]
+"func-tests/**/*.py" = [
+    # Use of assert detected — this is pytest
+    "S101",
+    # subprocess call: check for execution of untrusted input
+    "S603",
+    # hardcoded passwords / tokens — fixtures only, not real creds
+    "S105", "S106",
+]
 
 [tool.ruff.lint.isort]
 force-single-line = true