From f1493268e88d22de3819e5e9ebc81eb127a66622 Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 24 Jun 2026 01:52:03 +0530 Subject: [PATCH 01/10] feat: sp, a command-line client for the Sample Platform REST API sp lets a developer or an AI agent investigate CI runs end-to-end from the terminal (no web frontend): list/show runs, summaries, classified failures, expected-vs-actual diffs, logs, errors, artifacts, samples, regression tests, auth, and a one-shot `investigate` triage command. Output defaults to JSON for agents, with a -o table human view. The API address is configurable via --base-url / SP_BASE_URL. Includes a test suite (pytest), lint/type configs (pycodestyle, pydocstyle, isort, mypy), and a GitHub Actions CI workflow. --- .github/workflows/ci.yml | 41 ++++++ .gitignore | 23 ++++ README.md | 71 ++++++++++ pyproject.toml | 22 +++ setup.cfg | 15 +++ sp_cli/__init__.py | 9 ++ sp_cli/__main__.py | 6 + sp_cli/banner.py | 54 ++++++++ sp_cli/classifier.py | 110 +++++++++++++++ sp_cli/client.py | 168 +++++++++++++++++++++++ sp_cli/commands/__init__.py | 1 + sp_cli/commands/auth.py | 47 +++++++ sp_cli/commands/investigate.py | 72 ++++++++++ sp_cli/commands/regression.py | 29 ++++ sp_cli/commands/run.py | 236 +++++++++++++++++++++++++++++++++ sp_cli/commands/sample.py | 47 +++++++ sp_cli/commands/system.py | 23 ++++ sp_cli/main.py | 51 +++++++ sp_cli/output.py | 140 +++++++++++++++++++ sp_cli/runner.py | 41 ++++++ sp_cli/triage.py | 85 ++++++++++++ tests/__init__.py | 1 + tests/test_classifier.py | 69 ++++++++++ tests/test_cli.py | 186 ++++++++++++++++++++++++++ tests/test_client.py | 90 +++++++++++++ tests/test_triage.py | 99 ++++++++++++++ uv.lock | 186 ++++++++++++++++++++++++++ 27 files changed, 1922 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 sp_cli/__init__.py create mode 100644 sp_cli/__main__.py create mode 100644 sp_cli/banner.py create mode 100644 sp_cli/classifier.py create mode 100644 sp_cli/client.py create mode 100644 sp_cli/commands/__init__.py create mode 100644 sp_cli/commands/auth.py create mode 100644 sp_cli/commands/investigate.py create mode 100644 sp_cli/commands/regression.py create mode 100644 sp_cli/commands/run.py create mode 100644 sp_cli/commands/sample.py create mode 100644 sp_cli/commands/system.py create mode 100644 sp_cli/main.py create mode 100644 sp_cli/output.py create mode 100644 sp_cli/runner.py create mode 100644 sp_cli/triage.py create mode 100644 tests/__init__.py create mode 100644 tests/test_classifier.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_client.py create mode 100644 tests/test_triage.py create mode 100644 uv.lock diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..aa2cfcb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13'] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: isort + run: isort . --check-only --diff + + - name: pycodestyle + run: pycodestyle . + + - name: pydocstyle + run: pydocstyle sp_cli + + - name: mypy + run: mypy sp_cli + + - name: pytest + run: pytest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..16683c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ + +# Virtual environments +venv/ +.venv/ +env/ + +# Tooling caches +.pytest_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +# OS / editor +.DS_Store +.idea/ +.vscode/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66e7063 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# sp — CCExtractor Sample Platform CLI + +`sp` is a command-line client for the [CCExtractor Sample Platform](https://github.com/CCExtractor/sample-platform) +REST API. It lets a developer **or an AI agent** investigate CI runs end-to-end +from the terminal — no web frontend required. + +Output defaults to **JSON** (ideal for agents and scripts), with a human-friendly +`-o table` view. + +## Install + +```bash +pip install -e . +``` + +This installs the `sp` command. + +## Configure + +`sp` needs to know where the API lives and (optionally) a bearer token: + +```bash +export SP_BASE_URL=https://sampleplatform.ccextractor.org/api/v1 # or your instance +export SP_API_TOKEN= # if the API requires auth +``` + +Both can also be passed per-command with `--base-url` and `--token`. + +## Usage + +```bash +sp # banner / help +sp health # API + dependency health +sp run ls # list CI runs +sp run summary # pass/fail summary for a run +sp run failures # failing tests, each auto-classified +sp run diff # expected-vs-actual diff for a result +sp run logs # raw run logs +sp investigate # one-shot triage: info + counts + classified failures +``` + +Add `-o table` to any command for a human-readable view (default is JSON): + +```bash +sp -o table investigate 9299 +``` + +### The classifier + +`sp` labels each failure with a stable code — `SEGFAULT`, `ABORT`, `TIMEOUT`, +`EXIT_CODE_MISMATCH`, `MISSING_OUTPUT`, `OUTPUT_DIFF`, `PASS` — so a person or an +agent gets a straight answer about *why* a test failed, without reading logs. + +## Development + +```bash +python -m venv .venv && source .venv/bin/activate +pip install -e ".[dev]" + +isort . --check-only # import order +pycodestyle . # style +pydocstyle sp_cli # docstrings +mypy sp_cli # types +pytest # tests +``` + +## Relationship to the platform + +`sp` is a **client**: it talks to the Sample Platform's REST API over HTTP. It is +deliberately kept in its own repository, separate from the platform server that +gets deployed on the VM. Point it at any deployment via `SP_BASE_URL`. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dd84551 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "sp-cli" +version = "0.1.0" +description = "AI-friendly CLI for the CCExtractor CI / Sample Platform" +requires-python = ">=3.10" +dependencies = ["click", "requests"] + +[project.optional-dependencies] +dev = ["pytest", "pycodestyle", "pydocstyle", "isort", "mypy"] + +[project.scripts] +sp = "sp_cli.main:cli" + +[tool.setuptools] +packages = ["sp_cli", "sp_cli.commands"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..54d197f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,15 @@ +[pycodestyle] +max-line-length = 120 +ignore = E701 +exclude = .git,.venv,venv,build,dist,*.egg-info + +[pydocstyle] +convention = numpy +add-ignore = D100,D104 + +[isort] +skip = .venv,venv,build,dist + +[mypy] +python_version = 3.10 +ignore_missing_imports = True diff --git a/sp_cli/__init__.py b/sp_cli/__init__.py new file mode 100644 index 0000000..c3179aa --- /dev/null +++ b/sp_cli/__init__.py @@ -0,0 +1,9 @@ +"""``sp`` — an AI-friendly command-line client for the CCExtractor Sample Platform. + +The CLI is a thin layer over the Sample Platform JSON API (``/api/v1``). It is +designed to be driven by AI agents as well as humans: it emits machine-readable +JSON by default and uses non-zero exit codes plus a consistent error envelope on +failure, so it can be scripted without screen-scraping the web UI. +""" + +__version__ = "0.1.0" diff --git a/sp_cli/__main__.py b/sp_cli/__main__.py new file mode 100644 index 0000000..d5e9b2e --- /dev/null +++ b/sp_cli/__main__.py @@ -0,0 +1,6 @@ +"""Allow the CLI to be run as ``python -m sp_cli``.""" + +from sp_cli.main import cli + +if __name__ == '__main__': + cli() diff --git a/sp_cli/banner.py b/sp_cli/banner.py new file mode 100644 index 0000000..d0cb6de --- /dev/null +++ b/sp_cli/banner.py @@ -0,0 +1,54 @@ +"""Branded welcome screen for the ``sp`` CLI. + +Shown only when ``sp`` is invoked with no subcommand. Never emitted on command +output, so machine consumers (agents parsing JSON) are unaffected. Colors are +applied via :func:`click.style` and are auto-stripped when output is piped. +""" + +import click + +from sp_cli import __version__ + +#: Figlet-style "sp" wordmark. +LOGO = r""" ___ _ __ + / __| '_ \ + \__ \ |_) | + |___/ .__/ + |_|""" + +_GROUPS = [ + ('TRIAGE', 'sp investigate ← one-shot: what failed and why'), + ('RUNS', 'sp run ls · show · summary · failures · results · result · diff · artifacts · logs · errors'), + ('SAMPLES', 'sp sample ls · show · history'), + ('TESTS', 'sp regression ls'), + ('SYSTEM', 'sp health · queue'), + ('AUTH', 'sp auth login · logout'), +] + +_EXAMPLES = [ + ('sp investigate 9299', 'triage a run end-to-end'), + ('sp run failures 9299', 'failing tests, each labeled with why'), + ('sp run diff 9299 137', 'expected-vs-actual diff (ids auto-resolved)'), +] + + +def show_welcome() -> None: + """Print the branded welcome screen (banner, command map, examples).""" + click.echo() + click.echo(click.style(LOGO, fg='cyan')) + click.echo(f" {click.style('CCExtractor CI', bold=True)} · AI-friendly CLI · v{__version__}") + click.echo(" drive CI investigations from the terminal — no UI, no HTML scraping") + click.echo() + + for name, line in _GROUPS: + click.echo(f" {click.style(name.ljust(8), fg='green', bold=True)} {line}") + click.echo() + + click.echo(f" {click.style('Examples', bold=True)}") + for command, note in _EXAMPLES: + click.echo(f" {command.ljust(28)} {click.style('# ' + note, fg='bright_black')}") + click.echo() + + click.echo(f" {click.style('Help', bold=True)} sp COMMAND --help" + f" {click.style('Config', bold=True)} SP_BASE_URL · SP_API_TOKEN") + click.echo() diff --git a/sp_cli/classifier.py b/sp_cli/classifier.py new file mode 100644 index 0000000..ec67ef9 --- /dev/null +++ b/sp_cli/classifier.py @@ -0,0 +1,110 @@ +"""Rule-based classification of regression-test failures into stable codes. + +Deterministic, no ML: maps the raw signals a test run exposes (exit code, +expected return code, output presence, pass-history) onto a small, stable +taxonomy so an agent can branch on *why* a test failed instead of parsing +prose. Platform differences are normalized — e.g. a segfault surfaces as ``139`` +on Linux and ``-1073741819`` (0xC0000005) on Windows; both classify as +``SEGFAULT``. + +Each classification returns a ``code`` (stable, machine-readable), a +``confidence`` (``high`` for unambiguous exit-code rules, ``medium`` for +output-based ones), a human ``reason``, and ``regression`` (True if the test was +passing before — a real regression; False if it never passed; None if unknown). +""" + +from typing import Any, Dict, Optional + +# --- Failure codes (stable; downstream tools may pin on these) --------------- +CODE_PASS = "PASS" +CODE_SEGFAULT = "SEGFAULT" +CODE_ABORT = "ABORT" +CODE_TIMEOUT = "TIMEOUT" +CODE_MISSING_OUTPUT = "MISSING_OUTPUT" +CODE_EXIT_CODE_MISMATCH = "EXIT_CODE_MISMATCH" +CODE_OUTPUT_DIFF = "OUTPUT_DIFF" +CODE_UNKNOWN = "UNKNOWN" + +# --- Exit codes that denote a crash, normalized across platforms ------------- +#: SIGSEGV (128+11) on Linux, raw -11, and 0xC0000005 access violation on Windows. +_SEGFAULT_CODES = frozenset({139, -11, -1073741819}) +#: SIGABRT (128+6) on Linux and raw -6. +_ABORT_CODES = frozenset({134, -6}) +#: `timeout` exit (124) and SIGTERM (143 / -15). +_TIMEOUT_CODES = frozenset({124, 143, -15}) + + +def classify(exit_code: Optional[int], expected_rc: Optional[int], *, + has_output_diff: bool = False, missing_output: bool = False, + has_ever_passed: Optional[bool] = None) -> Dict[str, Any]: + """ + Classify a single regression-test result into a stable failure code. + + Rules are evaluated most-severe first (crash > timeout > missing output > + exit-code mismatch > output diff), so the most actionable signal wins. + + :param exit_code: The process exit code observed for the test. + :type exit_code: Optional[int] + :param expected_rc: The exit code the test was expected to return. + :type expected_rc: Optional[int] + :param has_output_diff: True if a differing output file was recorded. + :type has_output_diff: bool + :param missing_output: True if output was expected but none was produced. + :type missing_output: bool + :param has_ever_passed: Whether this test has ever passed (history), if known. + :type has_ever_passed: Optional[bool] + :return: ``{code, confidence, reason, regression}``. + :rtype: Dict[str, Any] + """ + regression = _regression_state(has_ever_passed) + + if exit_code in _SEGFAULT_CODES: + return _result(CODE_SEGFAULT, "high", + f"Crash (segfault / access violation), exit {exit_code}", regression) + if exit_code in _ABORT_CODES: + return _result(CODE_ABORT, "high", f"Aborted (SIGABRT), exit {exit_code}", regression) + if exit_code in _TIMEOUT_CODES: + return _result(CODE_TIMEOUT, "high", f"Timed out / terminated, exit {exit_code}", regression) + if missing_output: + return _result(CODE_MISSING_OUTPUT, "high", + "No output was produced but one was expected", regression) + if exit_code != expected_rc: + return _result(CODE_EXIT_CODE_MISMATCH, "high", + f"Exited {exit_code}, expected {expected_rc}", regression) + if has_output_diff: + return _result(CODE_OUTPUT_DIFF, "medium", + "Exit code matched but output differs from expected", regression) + + return _result(CODE_PASS, "high", "Exit code matched and no output diff recorded", regression) + + +def _regression_state(has_ever_passed: Optional[bool]) -> Optional[bool]: + """ + Translate pass-history into the ``regression`` flag. + + :param has_ever_passed: Whether the test has ever passed, if known. + :type has_ever_passed: Optional[bool] + :return: True if a real regression, False if never worked, None if unknown. + :rtype: Optional[bool] + """ + if has_ever_passed is None: + return None + return bool(has_ever_passed) + + +def _result(code: str, confidence: str, reason: str, regression: Optional[bool]) -> Dict[str, Any]: + """ + Assemble a classification result dict. + + :param code: The stable failure code. + :type code: str + :param confidence: ``high`` or ``medium``. + :type confidence: str + :param reason: Human-readable explanation. + :type reason: str + :param regression: Regression flag (see :func:`_regression_state`). + :type regression: Optional[bool] + :return: The assembled result. + :rtype: Dict[str, Any] + """ + return {"code": code, "confidence": confidence, "reason": reason, "regression": regression} diff --git a/sp_cli/client.py b/sp_cli/client.py new file mode 100644 index 0000000..2719dee --- /dev/null +++ b/sp_cli/client.py @@ -0,0 +1,168 @@ +"""HTTP client for the CCExtractor CI System API (`/api/v1`).""" + +from typing import Any, Dict, List, Optional + +import requests # type: ignore[import-untyped] + + +class ApiError(Exception): + """Raised when an API request fails, carrying the structured error envelope.""" + + def __init__(self, code: str, message: str, status: Optional[int] = None, + details: Optional[Dict[str, Any]] = None) -> None: + """ + Build an API error. + + :param code: Stable machine-readable error code (e.g. ``not_found``). + :type code: str + :param message: Human-readable explanation. + :type message: str + :param status: HTTP status code, if the failure was an HTTP response. + :type status: Optional[int] + :param details: Optional structured context echoed from the API. + :type details: Optional[Dict[str, Any]] + """ + super().__init__(message) + self.code = code + self.message = message + self.status = status + self.details = details + + @property + def exit_code(self) -> int: + """ + Map the error to a process exit code so callers can branch on it. + + :return: 3 connection · 4 not-found · 5 validation · 6 auth · 7 rate-limited · 1 other. + :rtype: int + """ + if self.code == 'connection_error': + return 3 + if self.status == 404: + return 4 + if self.status in (400, 422): + return 5 + if self.status in (401, 403): + return 6 + if self.status == 429: + return 7 + return 1 + + +class ApiClient: + """Minimal client over the JSON API. Sends a bearer token when configured.""" + + def __init__(self, base_url: str, token: Optional[str] = None, timeout: int = 30) -> None: + """ + Configure the client. + + :param base_url: Root URL of the platform (without the ``/api/v1`` prefix). + :type base_url: str + :param token: Optional opaque bearer token sent on every request. + :type token: Optional[str] + :param timeout: Per-request timeout in seconds. + :type timeout: int + """ + self.base_url = base_url.rstrip('/') + self.token = token + self.timeout = timeout + self.session = requests.Session() + + def _headers(self) -> Dict[str, str]: + """ + Build request headers, including the bearer token when set. + + :return: Header mapping. + :rtype: Dict[str, str] + """ + headers = {'Accept': 'application/json'} + if self.token: + headers['Authorization'] = f'Bearer {self.token}' + return headers + + def request(self, method: str, path: str, params: Optional[Dict[str, Any]] = None, + json_body: Optional[Dict[str, Any]] = None) -> Any: + """ + Perform a request against an API path and return the decoded JSON body. + + :param method: HTTP method (``GET``, ``POST``, ``DELETE`` …). + :type method: str + :param path: API path below ``/api/v1`` (e.g. ``/runs``). + :type path: str + :param params: Optional query-string parameters. + :type params: Optional[Dict[str, Any]] + :param json_body: Optional JSON request body. + :type json_body: Optional[Dict[str, Any]] + :raises ApiError: on connection failure, a non-JSON body, or an HTTP error. + :return: The decoded JSON response body (or ``None`` for ``204``). + :rtype: Any + """ + url = f"{self.base_url}{path}" + try: + response = self.session.request(method, url, params=params, json=json_body, + headers=self._headers(), timeout=self.timeout) + except requests.RequestException as exc: + raise ApiError('connection_error', f'Could not reach {url}: {exc}') + + if response.status_code == 204: + return None + + try: + payload = response.json() + except ValueError: + raise ApiError('invalid_response', + f'Expected JSON but got HTTP {response.status_code}', response.status_code) + + if response.status_code >= 400: + error = payload if isinstance(payload, dict) else {} + raise ApiError( + error.get('code', 'http_error'), + error.get('message', f'Request failed with HTTP {response.status_code}'), + response.status_code, + error.get('details'), + ) + + return payload + + def get(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: + """ + Perform a GET and return the decoded body. + + :param path: API path below ``/api/v1``. + :type path: str + :param params: Optional query-string parameters. + :type params: Optional[Dict[str, Any]] + :return: The decoded JSON body. + :rtype: Any + """ + return self.request('GET', path, params=params) + + def get_paginated(self, path: str, params: Optional[Dict[str, Any]] = None, + max_items: int = 1000) -> List[Any]: + """ + Follow offset pagination and return the combined ``data`` list. + + :param path: API path below ``/api/v1``. + :type path: str + :param params: Optional query-string parameters (``limit``/``offset`` are managed). + :type params: Optional[Dict[str, Any]] + :param max_items: Safety cap on total items collected. + :type max_items: int + :return: All items across pages. + :rtype: List[Any] + """ + merged = dict(params or {}) + merged.setdefault('limit', 100) + offset = 0 + items: List[Any] = [] + while True: + merged['offset'] = offset + payload = self.get(path, params=merged) + data = payload.get('data', []) if isinstance(payload, dict) else [] + items.extend(data) + pagination = payload.get('pagination', {}) if isinstance(payload, dict) else {} + next_offset = pagination.get('next_offset') + if not data or next_offset is None or len(items) >= max_items: + break + offset = next_offset + return items diff --git a/sp_cli/commands/__init__.py b/sp_cli/commands/__init__.py new file mode 100644 index 0000000..38d1226 --- /dev/null +++ b/sp_cli/commands/__init__.py @@ -0,0 +1 @@ +"""Command groups for the ``sp`` CLI, grouped by resource (noun-verb).""" diff --git a/sp_cli/commands/auth.py b/sp_cli/commands/auth.py new file mode 100644 index 0000000..3de56a8 --- /dev/null +++ b/sp_cli/commands/auth.py @@ -0,0 +1,47 @@ +"""``sp auth`` — obtain and revoke API tokens.""" + +import click + +from sp_cli.client import ApiError +from sp_cli.output import render, render_error + + +@click.group() +def auth() -> None: + """Obtain and revoke API tokens.""" + + +@auth.command('login') +@click.option('--email', prompt=True, help='Account email.') +@click.option('--password', prompt=True, hide_input=True, help='Account password (never stored).') +@click.option('--name', 'token_name', default='sp-cli', show_default=True, help='Token label.') +@click.option('--days', 'expires_in_days', type=int, default=30, show_default=True, + help='Token lifetime in days (max 90).') +@click.pass_context +def auth_login(ctx: click.Context, email: str, password: str, + token_name: str, expires_in_days: int) -> None: + """Create an API token; store the printed value in SP_API_TOKEN.""" + client = ctx.obj['client'] + output = ctx.obj['output'] + body = {'email': email, 'password': password, + 'token_name': token_name, 'expires_in_days': expires_in_days} + try: + result = client.request('POST', '/auth/tokens', json_body=body) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + render(result, output) + + +@auth.command('logout') +@click.pass_context +def auth_logout(ctx: click.Context) -> None: + """Revoke the current API token.""" + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + client.request('DELETE', '/auth/tokens/current') + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + click.echo('Token revoked.') diff --git a/sp_cli/commands/investigate.py b/sp_cli/commands/investigate.py new file mode 100644 index 0000000..16c2a24 --- /dev/null +++ b/sp_cli/commands/investigate.py @@ -0,0 +1,72 @@ +"""``sp investigate`` — one-shot triage of a run (status + counts + classified failures).""" + +from typing import Any, Dict, List + +import click + +from sp_cli.client import ApiError +from sp_cli.output import render, render_error +from sp_cli.triage import classify_sample, group_by_code, is_failure + +_RUN_FIELDS = ('run_id', 'pr_number', 'platform', 'commit_sha', 'branch', 'status', 'github_link') + + +@click.command('investigate') +@click.argument('run_id', type=int) +@click.pass_context +def investigate(ctx: click.Context, run_id: int) -> None: + """Triage a run in one shot: run info, pass/fail counts, and classified failures. + + Combines the run detail, summary, and per-result classification into a single + report — the whole "what failed and why" investigation in one command. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + run = client.get(f'/runs/{run_id}') + summary = client.get(f'/runs/{run_id}/summary') + samples = client.get_paginated(f'/runs/{run_id}/samples') + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + + failures = [classify_sample(s) for s in samples if is_failure(s)] + report = { + 'run': {field: run.get(field) for field in _RUN_FIELDS}, + 'summary': summary, + 'by_code': group_by_code(failures), + 'failures': failures, + } + + if output == 'json': + render(report, 'json') + else: + _print_digest(report) + + +def _print_digest(report: Dict[str, Any]) -> None: + """ + Print a human-readable investigation digest. + + :param report: The assembled investigation report. + :type report: Dict[str, Any] + """ + run = report['run'] + summary = report['summary'] + header = (f"Run {run.get('run_id')} · PR {run.get('pr_number')} · {run.get('platform')} · " + f"{run.get('commit_sha')} · {str(run.get('status')).upper()}") + click.echo(header) + click.echo(f" {summary.get('fail_count')} failed / {summary.get('total_samples')} total" + f" ({summary.get('pass_count')} pass)") + + by_code = report['by_code'] + if by_code: + click.echo() + click.echo(" by code:") + for code, count in by_code.items(): + click.echo(f" {str(count).rjust(4)} {code}") + + failures: List[Dict[str, Any]] = report['failures'] + if failures: + click.echo() + render({'data': failures}, 'table') diff --git a/sp_cli/commands/regression.py b/sp_cli/commands/regression.py new file mode 100644 index 0000000..fac9e37 --- /dev/null +++ b/sp_cli/commands/regression.py @@ -0,0 +1,29 @@ +"""``sp regression`` — list regression-test definitions.""" + +from typing import Optional + +import click + +from sp_cli.runner import clean_params, fetch_and_render + + +@click.group() +def regression() -> None: + """List regression-test definitions.""" + + +@regression.command('ls') +@click.option('--category', default=None, help='Filter by category name.') +@click.option('--tag', default=None, help='Filter by tag.') +@click.option('--active/--all', 'active', default=None, help='Only active tests (default: all).') +@click.option('--sample-id', type=int, default=None, help='Filter by sample id.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def regression_ls(ctx: click.Context, category: Optional[str], tag: Optional[str], + active: Optional[bool], sample_id: Optional[int], + limit: Optional[int], offset: Optional[int]) -> None: + """List regression-test definitions.""" + params = clean_params({'category': category, 'tag': tag, 'active': active, + 'sample_id': sample_id, 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, '/regression-tests', params) diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py new file mode 100644 index 0000000..b304248 --- /dev/null +++ b/sp_cli/commands/run.py @@ -0,0 +1,236 @@ +"""``sp run`` — list, inspect, and triage CI runs.""" + +from typing import Any, Dict, List, Optional, Tuple + +import click + +from sp_cli.client import ApiError +from sp_cli.output import render, render_error +from sp_cli.runner import clean_params, fetch_and_render +from sp_cli.triage import classify_sample, is_failure + + +@click.group() +def run() -> None: + """List, inspect, and triage CI runs.""" + + +@run.command('ls') +@click.option('--status', default=None, help='queued|running|pass|fail|canceled|error|incomplete') +@click.option('--platform', default=None, help='linux|windows') +@click.option('--branch', default=None, help='Filter by branch name.') +@click.option('--commit', 'commit_sha', default=None, help='Full 40-char commit SHA.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def run_ls(ctx: click.Context, status: Optional[str], platform: Optional[str], branch: Optional[str], + commit_sha: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: + """List CI runs (newest first).""" + params = clean_params({'status': status, 'platform': platform, 'branch': branch, + 'commit_sha': commit_sha, 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, '/runs', params) + + +@run.command('show') +@click.argument('run_id', type=int) +@click.pass_context +def run_show(ctx: click.Context, run_id: int) -> None: + """Show a single run's details.""" + fetch_and_render(ctx, f'/runs/{run_id}') + + +@run.command('summary') +@click.argument('run_id', type=int) +@click.pass_context +def run_summary(ctx: click.Context, run_id: int) -> None: + """Show a run's pass/fail summary.""" + fetch_and_render(ctx, f'/runs/{run_id}/summary') + + +@run.command('failures') +@click.argument('run_id', type=int) +@click.pass_context +def run_failures(ctx: click.Context, run_id: int) -> None: + """Show a run's failing tests, each labeled with a classification code.""" + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + samples = client.get_paginated(f'/runs/{run_id}/samples') + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + + rows = [classify_sample(s) for s in samples if is_failure(s)] + render({'data': rows, 'summary': {'failures': len(rows), 'of_total': len(samples)}}, output) + + +@run.command('results') +@click.argument('run_id', type=int) +@click.option('--status', default=None, help='pass|fail|skipped|missing_output|running|not_started') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def run_results(ctx: click.Context, run_id: int, status: Optional[str], + limit: Optional[int], offset: Optional[int]) -> None: + """List all regression-test results in a run.""" + params = clean_params({'status': status, 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, f'/runs/{run_id}/samples', params) + + +@run.command('result') +@click.argument('run_id', type=int) +@click.argument('sample_id', type=int) +@click.pass_context +def run_result(ctx: click.Context, run_id: int, sample_id: int) -> None: + """Show full details for a single regression-test result in a run.""" + fetch_and_render(ctx, f'/runs/{run_id}/samples/{sample_id}') + + +@run.command('diff') +@click.argument('run_id', type=int) +@click.argument('sample_id', type=int) +@click.option('--regression', 'regression_id', type=int, default=None, + help='Regression test id (auto-resolved if omitted).') +@click.option('--output', 'output_id', type=int, default=None, + help='Output file id (auto-resolved if omitted).') +@click.option('--context', 'context_lines', type=int, default=None, help='Diff context lines.') +@click.pass_context +def run_diff(ctx: click.Context, run_id: int, sample_id: int, regression_id: Optional[int], + output_id: Optional[int], context_lines: Optional[int]) -> None: + """Show the expected-vs-actual diff for a failing result. + + Resolves the (regression, output) ids automatically when omitted, so you + don't need the hidden ids the web UI requires to build a diff URL. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + targets = _resolve_diff_targets(client, run_id, sample_id, regression_id, output_id) + if not targets: + raise ApiError('not_found', 'No differing output to diff for this result', 404) + diffs = [] + for media_sample_id, reg_id, out_id in targets: + params = clean_params({'context_lines': context_lines}) + diffs.append(client.get( + f'/runs/{run_id}/samples/{media_sample_id}' + f'/regression-tests/{reg_id}/outputs/{out_id}/diff', + params=params)) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + + render(diffs[0] if len(diffs) == 1 else {'data': diffs}, output) + + +@run.command('approve-baseline') +@click.argument('run_id', type=int) +@click.argument('sample_id', type=int) +@click.option('--regression', 'regression_id', type=int, required=True, + help='Regression test id of the result to approve.') +@click.option('--output', 'output_id', type=int, required=True, + help="Output file id whose actual output becomes the new baseline.") +@click.option('--remove-variants', is_flag=True, default=False, + help='Remove all platform-specific variants (see WARNING below).') +@click.pass_context +def run_approve_baseline(ctx: click.Context, run_id: int, sample_id: int, + regression_id: int, output_id: int, remove_variants: bool) -> None: + """Approve a result's actual output as the new expected baseline. + + Requires admin privileges (the ``baselines:write`` scope). + + WARNING: --remove-variants makes this output the single source of truth + across ALL platforms by deleting every platform-specific variant. This + applies globally and cannot be undone from the CLI -- use with care. + + --regression and --output are required (no auto-resolution): approving a + baseline is destructive, so the exact target must be stated explicitly. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + # The endpoint's path slot is the *media* sample id, which + # differs from the regression-result id passed on the command line. + # Resolve it from the result detail (same contract as `run diff`). + detail = client.get(f'/runs/{run_id}/samples/{sample_id}') + media_sample_id = detail.get('sample_id') + if media_sample_id is None: + raise ApiError('not_found', 'Could not resolve the media sample for this result', 404) + body = {'regression_id': regression_id, 'output_id': output_id, + 'remove_variants': remove_variants} + result = client.request( + 'POST', f'/runs/{run_id}/samples/{media_sample_id}/baseline-approval', + json_body=body) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + + render(result, output) + + +@run.command('artifacts') +@click.argument('run_id', type=int) +@click.pass_context +def run_artifacts(ctx: click.Context, run_id: int) -> None: + """List downloadable artifacts for a run (signed URLs).""" + fetch_and_render(ctx, f'/runs/{run_id}/artifacts') + + +@run.command('logs') +@click.argument('run_id', type=int) +@click.pass_context +def run_logs(ctx: click.Context, run_id: int) -> None: + """Show raw logs for a run (requires contributor or admin privileges).""" + fetch_and_render(ctx, f'/runs/{run_id}/logs') + + +@run.command('errors') +@click.argument('run_id', type=int) +@click.option('--type', 'error_type', default=None, + help='test_failure|exit_code_mismatch|missing_output|diff_mismatch') +@click.pass_context +def run_errors(ctx: click.Context, run_id: int, error_type: Optional[str]) -> None: + """Show structured test errors for a run.""" + fetch_and_render(ctx, f'/runs/{run_id}/errors', clean_params({'type': error_type})) + + +def _resolve_diff_targets(client: Any, run_id: int, sample_id: int, + regression_id: Optional[int], + output_id: Optional[int]) -> List[Tuple[int, int, int]]: + """ + Resolve the (media_sample_id, regression_id, output_id) triples to diff. + + The diff endpoint is keyed by the *media* sample id, the regression test + id, and the output file id -- three different numbers. The CLI's + ``sample_id`` argument is the result id within the run (the regression test + id), so we always fetch the result detail to recover the media sample id + (``detail['sample_id']``) and the differing output(s), and let the caller + pass explicit ids only to narrow which output(s) to diff. + + :param client: The API client. + :type client: Any + :param run_id: The run id. + :type run_id: int + :param sample_id: The result id within the run (the regression test id). + :type sample_id: int + :param regression_id: Optional explicit regression id override. + :type regression_id: Optional[int] + :param output_id: Optional explicit output id to diff. + :type output_id: Optional[int] + :return: A list of ``(media_sample_id, regression_id, output_id)`` triples. + :rtype: List[Tuple[int, int, int]] + """ + detail = client.get(f'/runs/{run_id}/samples/{sample_id}') + media_sample_id = detail.get('sample_id') + reg_id = regression_id if regression_id is not None else detail.get('regression_test_id') + if media_sample_id is None or reg_id is None: + return [] + + outputs = detail.get('outputs') or [] + # The API reports per-output status as pass|fail|missing_output; anything + # not 'pass' is worth diffing. Fall back to all outputs if none qualify. + differing = [o for o in outputs if o.get('status') not in (None, 'pass')] or outputs + + if output_id is not None: + return [(media_sample_id, reg_id, output_id)] + return [(media_sample_id, reg_id, o.get('output_id')) for o in differing + if o.get('output_id') is not None] diff --git a/sp_cli/commands/sample.py b/sp_cli/commands/sample.py new file mode 100644 index 0000000..d8bd8ae --- /dev/null +++ b/sp_cli/commands/sample.py @@ -0,0 +1,47 @@ +"""``sp sample`` — list and inspect media samples.""" + +from typing import Optional + +import click + +from sp_cli.runner import clean_params, fetch_and_render + + +@click.group() +def sample() -> None: + """List and inspect media samples.""" + + +@sample.command('ls') +@click.option('--name', default=None, help='Filter by sample name.') +@click.option('--tag', default=None, help='Filter by tag.') +@click.option('--extension', default=None, help='Filter by file extension.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def sample_ls(ctx: click.Context, name: Optional[str], tag: Optional[str], + extension: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: + """List known media samples.""" + params = clean_params({'name': name, 'tag': tag, 'extension': extension, + 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, '/samples', params) + + +@sample.command('show') +@click.argument('sample_id', type=int) +@click.pass_context +def sample_show(ctx: click.Context, sample_id: int) -> None: + """Show metadata for a single sample.""" + fetch_and_render(ctx, f'/samples/{sample_id}') + + +@sample.command('history') +@click.argument('sample_id', type=int) +@click.option('--platform', default=None, help='linux|windows') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.pass_context +def sample_history(ctx: click.Context, sample_id: int, platform: Optional[str], + limit: Optional[int]) -> None: + """Show this sample's result history across runs.""" + params = clean_params({'platform': platform, 'limit': limit}) + fetch_and_render(ctx, f'/samples/{sample_id}/history', params) diff --git a/sp_cli/commands/system.py b/sp_cli/commands/system.py new file mode 100644 index 0000000..cb913a2 --- /dev/null +++ b/sp_cli/commands/system.py @@ -0,0 +1,23 @@ +"""``sp health`` and ``sp queue`` — system status commands.""" + +from typing import Optional + +import click + +from sp_cli.runner import clean_params, fetch_and_render + + +@click.command('health') +@click.pass_context +def health(ctx: click.Context) -> None: + """Show CI system health and dependency status.""" + fetch_and_render(ctx, '/system/health') + + +@click.command('queue') +@click.option('--platform', default=None, help='linux|windows') +@click.option('--status', default=None, help='queued|running') +@click.pass_context +def queue(ctx: click.Context, platform: Optional[str], status: Optional[str]) -> None: + """Show queue depth and currently running jobs.""" + fetch_and_render(ctx, '/system/queue', clean_params({'platform': platform, 'status': status})) diff --git a/sp_cli/main.py b/sp_cli/main.py new file mode 100644 index 0000000..47d8c14 --- /dev/null +++ b/sp_cli/main.py @@ -0,0 +1,51 @@ +"""Entry point and root command group for the ``sp`` CLI.""" + +from typing import Optional + +import click + +from sp_cli import __version__ +from sp_cli.client import ApiClient +from sp_cli.commands.auth import auth +from sp_cli.commands.investigate import investigate +from sp_cli.commands.regression import regression +from sp_cli.commands.run import run +from sp_cli.commands.sample import sample +from sp_cli.commands.system import health, queue + +DEFAULT_BASE_URL = 'http://localhost:5000/api/v1' + + +@click.group(invoke_without_command=True) +@click.option('--base-url', envvar='SP_BASE_URL', default=DEFAULT_BASE_URL, show_default=True, + help='API base URL incl. the /api/v1 prefix. Env: SP_BASE_URL.') +@click.option('--token', envvar='SP_API_TOKEN', default=None, + help='Bearer token sent with each request. Env: SP_API_TOKEN.') +@click.option('--output', '-o', type=click.Choice(['json', 'table']), default='json', show_default=True, + help='Output format.') +@click.option('--timeout', type=int, default=30, show_default=True, help='Per-request timeout (seconds).') +@click.version_option(__version__, prog_name='sp') +@click.pass_context +def cli(ctx: click.Context, base_url: str, token: Optional[str], output: str, timeout: int) -> None: + """AI-friendly CLI for the CCExtractor CI / Sample Platform. + + Emits JSON by default so it can be driven by agents and scripts. Point it at + a running platform with --base-url or the SP_BASE_URL environment variable, + and authenticate with a token via --token / SP_API_TOKEN (see `sp auth login`). + """ + ctx.obj = { + 'client': ApiClient(base_url, token=token, timeout=timeout), + 'output': output, + } + if ctx.invoked_subcommand is None: + from sp_cli.banner import show_welcome + show_welcome() + + +cli.add_command(investigate) +cli.add_command(run) +cli.add_command(sample) +cli.add_command(regression) +cli.add_command(auth) +cli.add_command(health) +cli.add_command(queue) diff --git a/sp_cli/output.py b/sp_cli/output.py new file mode 100644 index 0000000..cd5e0d2 --- /dev/null +++ b/sp_cli/output.py @@ -0,0 +1,140 @@ +"""Render API responses to the terminal as JSON (default) or a simple table.""" + +import json +from typing import Any, Dict, List + +import click + +from sp_cli.client import ApiError + +#: Value types rendered as plain table columns; nested structures are skipped. +_SCALAR = (str, int, float, bool, type(None)) + + +def render(payload: Any, output: str) -> None: + """ + Render a successful API payload in the requested format. + + Handles the API's three shapes: a list wrapper (``{data, pagination}``), a + flat single object (run/summary/health), and bare values. + + :param payload: The decoded JSON body returned by the API. + :type payload: Any + :param output: Either ``json`` or ``table``. + :type output: str + """ + if output == 'json': + click.echo(json.dumps(payload, indent=2)) + return + + if isinstance(payload, dict) and isinstance(payload.get('data'), list): + _print_rows(payload['data']) + footer = _footer(payload) + if footer: + click.echo(f"\n{footer}") + elif isinstance(payload, dict): + _print_kv(payload) + else: + click.echo(json.dumps(payload, indent=2)) + + +def render_error(error: ApiError, output: str) -> None: + """ + Render an API error as a JSON envelope on stderr, regardless of output mode. + + :param error: The error to render. + :type error: ApiError + :param output: The selected output mode (unused; kept for symmetry). + :type output: str + """ + envelope: Dict[str, Any] = {'error': {'code': error.code, 'message': error.message}} + if error.status is not None: + envelope['error']['status'] = error.status + if error.details: + envelope['error']['details'] = error.details + click.echo(json.dumps(envelope, indent=2), err=True) + + +def _footer(payload: Dict[str, Any]) -> str: + """ + Build a one-line footer from a ``summary`` or ``pagination`` block. + + :param payload: The full response payload. + :type payload: Dict[str, Any] + :return: A footer string (possibly empty). + :rtype: str + """ + summary = payload.get('summary') + if isinstance(summary, dict): + return ' · '.join(f"{k}: {v}" for k, v in summary.items()) + pagination = payload.get('pagination') + if isinstance(pagination, dict): + parts = [] + if pagination.get('total') is not None: + parts.append(f"{pagination['total']} total") + if pagination.get('next_offset') is not None: + parts.append(f"more at offset {pagination['next_offset']}") + return ' · '.join(parts) + return '' + + +def _print_rows(rows: List[Any]) -> None: + """ + Print a list of flat dicts as an aligned table of their scalar fields. + + :param rows: The list of row dicts to render. + :type rows: List[Any] + """ + if not rows: + click.echo('(no results)') + return + if not all(isinstance(row, dict) for row in rows): + click.echo(json.dumps(rows, indent=2)) + return + + columns: List[str] = [] + for row in rows: + for key, value in row.items(): + if key not in columns and isinstance(value, _SCALAR): + columns.append(key) + + widths = {col: len(col) for col in columns} + for row in rows: + for col in columns: + widths[col] = max(widths[col], len(_cell(row.get(col)))) + + click.echo(' '.join(col.ljust(widths[col]) for col in columns)) + click.echo(' '.join('-' * widths[col] for col in columns)) + for row in rows: + click.echo(' '.join(_cell(row.get(col)).ljust(widths[col]) for col in columns)) + + +def _print_kv(record: Dict[str, Any]) -> None: + """ + Print a single record as ``key: value`` lines, JSON-encoding nested values. + + :param record: The record to render. + :type record: Dict[str, Any] + """ + width = max((len(key) for key in record), default=0) + for key, value in record.items(): + rendered = _cell(value) if isinstance(value, _SCALAR) else json.dumps(value) + click.echo(f"{key.ljust(width)} : {rendered}") + + +def _cell(value: Any) -> str: + """ + Format a scalar cell value for table display. + + :param value: The value to format. + :type value: Any + :return: A string representation (empty string for ``None``). + :rtype: str + """ + if value is None: + return '' + if isinstance(value, bool): + return 'true' if value else 'false' + if isinstance(value, (list, tuple)): + return ', '.join(str(item) for item in value) + return str(value) diff --git a/sp_cli/runner.py b/sp_cli/runner.py new file mode 100644 index 0000000..5276772 --- /dev/null +++ b/sp_cli/runner.py @@ -0,0 +1,41 @@ +"""Shared helper that fetches from the API and renders the result or error.""" + +from typing import Any, Dict, Optional + +import click + +from sp_cli.client import ApiError +from sp_cli.output import render, render_error + + +def fetch_and_render(ctx: click.Context, path: str, params: Optional[Dict[str, Any]] = None) -> None: + """ + Fetch a path via the configured client and render it, exiting on error. + + :param ctx: The active Click context (carries the client and output mode). + :type ctx: click.Context + :param path: API path below ``/api/v1``. + :type path: str + :param params: Optional query-string parameters. + :type params: Optional[Dict[str, Any]] + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + payload = client.get(path, params=params) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + render(payload, output) + + +def clean_params(params: Dict[str, Any]) -> Dict[str, Any]: + """ + Drop ``None`` values so unset options are not sent as query parameters. + + :param params: Raw mapping of option names to values. + :type params: Dict[str, Any] + :return: The mapping without ``None`` values. + :rtype: Dict[str, Any] + """ + return {key: value for key, value in params.items() if value is not None} diff --git a/sp_cli/triage.py b/sp_cli/triage.py new file mode 100644 index 0000000..4eaec6e --- /dev/null +++ b/sp_cli/triage.py @@ -0,0 +1,85 @@ +"""Triage helpers: turn raw RunSample results into classified failure rows. + +Shared by ``sp run failures`` and ``sp investigate`` so both label failures the +same way. The classification itself lives in :mod:`sp_cli.classifier`; this module +adapts a ``RunSample`` (from ``/runs/{id}/samples``) into a flat, agent-friendly row. +""" + +from typing import Any, Dict, List + +from sp_cli.classifier import classify + +#: RunSample statuses that count as a failure worth triaging. +FAILURE_STATUSES = ('fail', 'missing_output') + + +def is_failure(sample: Dict[str, Any]) -> bool: + """ + Report whether a RunSample result is a failure. + + :param sample: One ``RunSample`` object. + :type sample: Dict[str, Any] + :return: True if the result failed or produced no output. + :rtype: bool + """ + return sample.get('status') in FAILURE_STATUSES + + +def has_output_diff(sample: Dict[str, Any]) -> bool: + """ + Decide whether a failing sample recorded a differing output file. + + Prefers the per-output ``status`` when present; otherwise falls back to + "failed but the exit code matched, so the failure must be an output diff." + + :param sample: One ``RunSample`` object. + :type sample: Dict[str, Any] + :return: True if an output differed from expected. + :rtype: bool + """ + outputs = sample.get('outputs') or [] + if outputs: + return any(item.get('status') not in (None, 'pass') for item in outputs) + return sample.get('status') == 'fail' and sample.get('exit_code') == sample.get('expected_rc') + + +def classify_sample(sample: Dict[str, Any]) -> Dict[str, Any]: + """ + Map a RunSample result onto a classified failure row. + + :param sample: One ``RunSample`` object from ``/runs/{id}/samples``. + :type sample: Dict[str, Any] + :return: A flat row with ids plus the classification code, confidence and reason. + :rtype: Dict[str, Any] + """ + label = classify( + sample.get('exit_code'), sample.get('expected_rc'), + missing_output=(sample.get('status') == 'missing_output'), + has_output_diff=has_output_diff(sample), + ) + return { + 'regression_test_id': sample.get('regression_test_id'), + 'sample_id': sample.get('sample_id'), + 'sample_name': sample.get('sample_name'), + 'categories': sample.get('categories') or [], + 'exit_code': sample.get('exit_code'), + 'expected_rc': sample.get('expected_rc'), + 'code': label['code'], + 'confidence': label['confidence'], + 'reason': label['reason'], + } + + +def group_by_code(failures: List[Dict[str, Any]]) -> Dict[str, int]: + """ + Count classified failures by their code. + + :param failures: Classified failure rows. + :type failures: List[Dict[str, Any]] + :return: A mapping of code → count, highest first. + :rtype: Dict[str, int] + """ + counts: Dict[str, int] = {} + for failure in failures: + counts[failure['code']] = counts.get(failure['code'], 0) + 1 + return dict(sorted(counts.items(), key=lambda item: item[1], reverse=True)) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..ccccefe --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the sp CLI (sp_cli).""" diff --git a/tests/test_classifier.py b/tests/test_classifier.py new file mode 100644 index 0000000..34df46c --- /dev/null +++ b/tests/test_classifier.py @@ -0,0 +1,69 @@ +"""Tests for the rule-based failure classifier, using real examples from run #9299.""" + +import unittest + +from sp_cli import classifier + + +class ClassifierTests(unittest.TestCase): + """Each case is grounded in a real failure observed in the friction study.""" + + def test_exit_code_mismatch(self): + """`10 (Expected 0)` — the common CEA-708 failure in run #9299.""" + result = classifier.classify(10, 0) + self.assertEqual(result["code"], classifier.CODE_EXIT_CODE_MISMATCH) + self.assertEqual(result["confidence"], "high") + self.assertIn("10", result["reason"]) + + def test_windows_segfault_normalized(self): + """`-1073741819` (0xC0000005) on Windows — the DVB failure in #9299.""" + result = classifier.classify(-1073741819, 0) + self.assertEqual(result["code"], classifier.CODE_SEGFAULT) + self.assertEqual(result["confidence"], "high") + + def test_linux_segfault_normalized(self): + """`139` on Linux is the same crash — must map to the same code.""" + self.assertEqual(classifier.classify(139, 0)["code"], classifier.CODE_SEGFAULT) + + def test_abort(self): + """`134` (SIGABRT) classifies as ABORT.""" + self.assertEqual(classifier.classify(134, 0)["code"], classifier.CODE_ABORT) + + def test_timeout(self): + """`124` (timeout) classifies as TIMEOUT.""" + self.assertEqual(classifier.classify(124, 0)["code"], classifier.CODE_TIMEOUT) + + def test_missing_output(self): + """'No output generated but there should be' — exit matches but output missing.""" + result = classifier.classify(0, 0, missing_output=True) + self.assertEqual(result["code"], classifier.CODE_MISSING_OUTPUT) + + def test_output_diff(self): + """Exit code matches but the output file differs.""" + result = classifier.classify(0, 0, has_output_diff=True) + self.assertEqual(result["code"], classifier.CODE_OUTPUT_DIFF) + self.assertEqual(result["confidence"], "medium") + + def test_pass(self): + """Exit matches, no diff, nothing missing → PASS.""" + self.assertEqual(classifier.classify(0, 0)["code"], classifier.CODE_PASS) + + def test_crash_beats_exit_mismatch(self): + """A segfault is reported as SEGFAULT, not a generic exit mismatch.""" + self.assertEqual(classifier.classify(139, 0)["code"], classifier.CODE_SEGFAULT) + + def test_regression_flag_true_when_previously_passed(self): + """A failure on a test that has passed before is a real regression.""" + self.assertTrue(classifier.classify(10, 0, has_ever_passed=True)["regression"]) + + def test_regression_flag_false_when_never_passed(self): + """A failure on a test that never passed is pre-existing (never worked).""" + self.assertFalse(classifier.classify(10, 0, has_ever_passed=False)["regression"]) + + def test_regression_flag_none_when_unknown(self): + """Without history, the regression flag is None (unknown).""" + self.assertIsNone(classifier.classify(10, 0)["regression"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..4836c57 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,186 @@ +"""Tests for the sp CLI command surface, mocking the API client.""" + +import json +import unittest +from unittest import mock + +from click.testing import CliRunner + +from sp_cli.client import ApiError +from sp_cli.main import cli + +RUNS_PAGE = { + 'data': [{'run_id': 9299, 'status': 'fail', 'platform': 'windows', 'commit_sha': 'e6cd34e'}], + 'pagination': {'limit': 50, 'offset': 0, 'total': 1, 'next_offset': None}, +} + +# A run's results: a segfault, an exit mismatch, a missing output, and a pass. +RUN_SAMPLES = [ + {'regression_test_id': 18, 'sample_name': 'dvb', 'categories': ['DVB'], + 'status': 'fail', 'exit_code': -1073741819, 'expected_rc': 0, 'outputs': []}, + {'regression_test_id': 137, 'sample_name': 'cea708', 'categories': ['CEA-708'], + 'status': 'fail', 'exit_code': 10, 'expected_rc': 0, 'outputs': []}, + {'regression_test_id': 7, 'sample_name': 'broken', 'categories': ['Broken'], + 'status': 'missing_output', 'exit_code': 0, 'expected_rc': 0, 'outputs': []}, + {'regression_test_id': 1, 'sample_name': 'ok', 'categories': ['General'], + 'status': 'pass', 'exit_code': 0, 'expected_rc': 0, 'outputs': []}, +] + + +class CliCommandTests(unittest.TestCase): + """Exercise the CLI commands with a mocked client.""" + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_ls_calls_runs_with_filters(self, mock_get): + """`run ls` hits /runs and forwards set filters only.""" + mock_get.return_value = RUNS_PAGE + result = self.runner.invoke(cli, ['run', 'ls', '--platform', 'windows']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs', params={'platform': 'windows'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_show(self, mock_get): + """`run show ` targets the run detail path.""" + mock_get.return_value = {'run_id': 9299, 'status': 'fail'} + result = self.runner.invoke(cli, ['run', 'show', '9299']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299', params=None) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + def test_run_failures_classifies(self, mock_paginated): + """`run failures` keeps only failures and labels each with a code.""" + mock_paginated.return_value = RUN_SAMPLES + result = self.runner.invoke(cli, ['run', 'failures', '9299']) + + self.assertEqual(result.exit_code, 0) + mock_paginated.assert_called_once_with('/runs/9299/samples') + data = json.loads(result.output) + codes = {row['regression_test_id']: row['code'] for row in data['data']} + self.assertEqual(codes, {18: 'SEGFAULT', 137: 'EXIT_CODE_MISMATCH', 7: 'MISSING_OUTPUT'}) + self.assertEqual(data['summary'], {'failures': 3, 'of_total': 4}) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + def test_run_failures_table_output(self, mock_paginated): + """Table mode renders the classification columns.""" + mock_paginated.return_value = RUN_SAMPLES + result = self.runner.invoke(cli, ['-o', 'table', 'run', 'failures', '9299']) + + self.assertEqual(result.exit_code, 0) + self.assertIn('SEGFAULT', result.output) + self.assertIn('code', result.output) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_sample_ls(self, mock_get): + """`sample ls` hits /samples.""" + mock_get.return_value = {'data': [], 'pagination': {'total': 0, 'next_offset': None}} + result = self.runner.invoke(cli, ['sample', 'ls', '--tag', 'teletext']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/samples', params={'tag': 'teletext'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_health(self, mock_get): + """`sp health` hits /system/health.""" + mock_get.return_value = {'status': 'ok', 'dependencies': []} + result = self.runner.invoke(cli, ['health']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/system/health', params=None) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_not_found_maps_to_exit_code_and_stderr(self, mock_get): + """A not-found error exits 4 with a JSON envelope on stderr.""" + mock_get.side_effect = ApiError('not_found', 'Run 9 not found', 404) + result = self.runner.invoke(cli, ['run', 'show', '9']) + + self.assertEqual(result.exit_code, 4) + self.assertEqual(json.loads(result.stderr)['error']['code'], 'not_found') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_result(self, mock_get): + """`run result ` targets the result-detail path.""" + mock_get.return_value = {'regression_test_id': 137, 'status': 'fail'} + result = self.runner.invoke(cli, ['run', 'result', '9299', '5']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/samples/5', params=None) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_diff_auto_resolves_hidden_ids(self, mock_get): + """`run diff` resolves the media sample id + regression/output ids from detail.""" + mock_get.side_effect = [ + {'regression_test_id': 137, 'sample_id': 42, + 'outputs': [{'output_id': 2, 'status': 'fail'}]}, + {'status': 'different', 'hunks': []}, + ] + result = self.runner.invoke(cli, ['run', 'diff', '9299', '5']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(mock_get.call_count, 2) + args, kwargs = mock_get.call_args + self.assertEqual(args[0], '/runs/9299/samples/42/regression-tests/137/outputs/2/diff') + self.assertEqual(kwargs['params'], {}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_diff_with_explicit_ids_uses_media_sample_from_detail(self, mock_get): + """Explicit --regression/--output still fetch detail for the media sample id.""" + mock_get.side_effect = [ + {'regression_test_id': 137, 'sample_id': 42, 'outputs': []}, + {'status': 'different', 'hunks': []}, + ] + result = self.runner.invoke(cli, ['run', 'diff', '9299', '5', '--regression', '137', '--output', '2']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(mock_get.call_count, 2) + args, kwargs = mock_get.call_args + self.assertEqual(args[0], '/runs/9299/samples/42/regression-tests/137/outputs/2/diff') + self.assertEqual(kwargs['params'], {}) + + @mock.patch('sp_cli.client.ApiClient.request') + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_approve_baseline_resolves_media_sample_and_posts(self, mock_get, mock_request): + """`run approve-baseline` POSTs to the media-sample path resolved from detail.""" + mock_get.return_value = {'regression_test_id': 137, 'sample_id': 42, 'outputs': []} + mock_request.return_value = {'status': 'approved', 'run_id': 9299, 'sample_id': 42, + 'regression_id': 137, 'output_id': 2} + result = self.runner.invoke(cli, ['run', 'approve-baseline', '9299', '5', + '--regression', '137', '--output', '2', '--remove-variants']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/samples/5') + mock_request.assert_called_once_with( + 'POST', '/runs/9299/samples/42/baseline-approval', + json_body={'regression_id': 137, 'output_id': 2, 'remove_variants': True}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_approve_baseline_requires_regression_and_output(self, mock_get): + """Approving a baseline refuses to run without the explicit target ids.""" + result = self.runner.invoke(cli, ['run', 'approve-baseline', '9299', '5']) + + self.assertNotEqual(result.exit_code, 0) + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_investigate_combines_run_summary_and_failures(self, mock_get, mock_paginated): + """`investigate` merges run detail, summary, and classified failures.""" + mock_get.side_effect = [ + {'run_id': 9299, 'pr_number': 2264, 'platform': 'windows', 'status': 'fail'}, + {'run_id': 9299, 'total_samples': 4, 'pass_count': 1, 'fail_count': 3}, + ] + mock_paginated.return_value = RUN_SAMPLES + result = self.runner.invoke(cli, ['investigate', '9299']) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.output) + self.assertEqual(report['run']['pr_number'], 2264) + self.assertEqual(report['summary']['fail_count'], 3) + self.assertEqual(report['by_code'], + {'SEGFAULT': 1, 'EXIT_CODE_MISMATCH': 1, 'MISSING_OUTPUT': 1}) + self.assertEqual(len(report['failures']), 3) diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..5eeeb5c --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,90 @@ +"""Tests for the CLI's HTTP client, mocking the requests session.""" + +import unittest +from unittest import mock + +import requests # type: ignore[import-untyped] + +from sp_cli.client import ApiClient, ApiError + + +class FakeResponse: + """Minimal stand-in for a requests Response.""" + + def __init__(self, status_code, json_data=None, raise_json=False): + """Store the canned status and body.""" + self.status_code = status_code + self._json = json_data + self._raise_json = raise_json + + def json(self): + """Return the canned JSON body or raise like requests does on non-JSON.""" + if self._raise_json: + raise ValueError('No JSON could be decoded') + return self._json + + +class ApiClientTests(unittest.TestCase): + """Exercise request building and error mapping in the client.""" + + @mock.patch('requests.Session.request') + def test_get_returns_payload_and_builds_url(self, mock_request): + """A 2xx response is returned and the API prefix is applied.""" + mock_request.return_value = FakeResponse(200, {'data': []}) + client = ApiClient('https://host/api/v1') + + self.assertEqual(client.get('/runs'), {'data': []}) + args, _ = mock_request.call_args + self.assertEqual(args[0], 'GET') + self.assertEqual(args[1], 'https://host/api/v1/runs') + + @mock.patch('requests.Session.request') + def test_204_returns_none(self, mock_request): + """A 204 (e.g. token revoke) returns None, not a parse error.""" + mock_request.return_value = FakeResponse(204) + self.assertIsNone(ApiClient('https://host').request('DELETE', '/auth/tokens/current')) + + @mock.patch('requests.Session.request') + def test_error_codes_map_to_exit_codes(self, mock_request): + """Each HTTP error maps to its documented exit code.""" + cases = {404: 4, 422: 5, 400: 5, 401: 6, 403: 6, 429: 7} + client = ApiClient('https://host') + for status, expected_exit in cases.items(): + mock_request.return_value = FakeResponse(status, {'code': 'x', 'message': 'm'}) + with self.assertRaises(ApiError) as caught: + client.get('/runs/9') + self.assertEqual(caught.exception.exit_code, expected_exit, f'status {status}') + + @mock.patch('requests.Session.request') + def test_token_is_sent_as_bearer_header(self, mock_request): + """A configured token is sent as an Authorization header.""" + mock_request.return_value = FakeResponse(200, {}) + ApiClient('https://host', token='secret').get('/runs') + _, kwargs = mock_request.call_args + self.assertEqual(kwargs['headers']['Authorization'], 'Bearer secret') + + @mock.patch('requests.Session.request', side_effect=requests.RequestException('boom')) + def test_connection_failure(self, mock_request): + """A transport failure maps to a connection_error with exit code 3.""" + with self.assertRaises(ApiError) as caught: + ApiClient('https://host').get('/runs') + self.assertEqual(caught.exception.code, 'connection_error') + self.assertEqual(caught.exception.exit_code, 3) + + @mock.patch('requests.Session.request') + def test_non_json_body_raises_invalid_response(self, mock_request): + """A non-JSON body raises invalid_response rather than crashing.""" + mock_request.return_value = FakeResponse(500, raise_json=True) + with self.assertRaises(ApiError) as caught: + ApiClient('https://host').get('/runs') + self.assertEqual(caught.exception.code, 'invalid_response') + + @mock.patch('requests.Session.request') + def test_get_paginated_follows_offset(self, mock_request): + """Pagination is followed across pages until next_offset is null.""" + mock_request.side_effect = [ + FakeResponse(200, {'data': [1, 2, 3], 'pagination': {'next_offset': 3}}), + FakeResponse(200, {'data': [4, 5], 'pagination': {'next_offset': None}}), + ] + items = ApiClient('https://host').get_paginated('/runs/9/samples') + self.assertEqual(items, [1, 2, 3, 4, 5]) diff --git a/tests/test_triage.py b/tests/test_triage.py new file mode 100644 index 0000000..5a570c9 --- /dev/null +++ b/tests/test_triage.py @@ -0,0 +1,99 @@ +"""Tests for the triage helpers that adapt RunSample results into failure rows.""" + +import unittest + +from sp_cli import triage + + +class IsFailureTests(unittest.TestCase): + """``is_failure`` keys off the RunSample status.""" + + def test_fail_status_is_failure(self): + """A 'fail' status counts as a failure worth triaging.""" + self.assertTrue(triage.is_failure({"status": "fail"})) + + def test_missing_output_is_failure(self): + """A 'missing_output' status counts as a failure.""" + self.assertTrue(triage.is_failure({"status": "missing_output"})) + + def test_pass_status_is_not_failure(self): + """A 'pass' status is not a failure.""" + self.assertFalse(triage.is_failure({"status": "pass"})) + + def test_missing_status_is_not_failure(self): + """A result with no status is not treated as a failure.""" + self.assertFalse(triage.is_failure({})) + + +class HasOutputDiffTests(unittest.TestCase): + """``has_output_diff`` prefers per-output status, matching the API's 'pass'.""" + + def test_passing_output_is_not_a_diff(self): + """A per-output status of 'pass' must not be reported as a diff.""" + sample = {"status": "fail", "outputs": [{"status": "pass"}]} + self.assertFalse(triage.has_output_diff(sample)) + + def test_failing_output_is_a_diff(self): + """A per-output status other than 'pass' is a differing output.""" + sample = {"status": "fail", "outputs": [{"status": "fail"}]} + self.assertTrue(triage.has_output_diff(sample)) + + def test_mixed_outputs_report_a_diff(self): + """If any output differs, the sample has an output diff.""" + sample = {"status": "fail", + "outputs": [{"status": "pass"}, {"status": "fail"}]} + self.assertTrue(triage.has_output_diff(sample)) + + def test_no_outputs_falls_back_to_matching_exit_code(self): + """Without per-output data, a fail whose exit code matched is a diff.""" + sample = {"status": "fail", "exit_code": 0, "expected_rc": 0} + self.assertTrue(triage.has_output_diff(sample)) + + def test_no_outputs_and_exit_mismatch_is_not_a_diff(self): + """Without per-output data, a fail with a mismatched exit code is not a diff.""" + sample = {"status": "fail", "exit_code": 1, "expected_rc": 0} + self.assertFalse(triage.has_output_diff(sample)) + + +class ClassifySampleTests(unittest.TestCase): + """``classify_sample`` flattens a RunSample into an agent-friendly row.""" + + def test_carries_ids_and_classification(self): + """The row carries the result ids plus the classification code.""" + sample = { + "regression_test_id": 42, + "sample_id": 7, + "sample_name": "dvb_subtitles", + "categories": ["DVB"], + "exit_code": 10, + "expected_rc": 0, + "status": "fail", + "outputs": [{"status": "pass"}], + } + row = triage.classify_sample(sample) + self.assertEqual(row["regression_test_id"], 42) + self.assertEqual(row["sample_id"], 7) + self.assertEqual(row["sample_name"], "dvb_subtitles") + self.assertEqual(row["categories"], ["DVB"]) + self.assertIn("code", row) + self.assertIn("confidence", row) + self.assertIn("reason", row) + + +class GroupByCodeTests(unittest.TestCase): + """``group_by_code`` counts failures by code, highest first.""" + + def test_counts_sorted_descending(self): + """Codes are returned ordered by frequency, most common first.""" + failures = [ + {"code": "EXIT_CODE_MISMATCH"}, + {"code": "SEGFAULT"}, + {"code": "EXIT_CODE_MISMATCH"}, + ] + counts = triage.group_by_code(failures) + self.assertEqual(counts, {"EXIT_CODE_MISMATCH": 2, "SEGFAULT": 1}) + self.assertEqual(next(iter(counts)), "EXIT_CODE_MISMATCH") + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..e24b0da --- /dev/null +++ b/uv.lock @@ -0,0 +1,186 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "certifi" +version = "2026.6.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, + { url = "https://files.pythonhosted.org/packages/24/47/b192933e94b546f1b1fe4df9cc1f84fcdbf2359f8d1081d46dd029b50207/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", size = 209329, upload-time = "2026-04-02T09:25:42.354Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/01fa81c5ca6141024d89a8fc15968002b71da7f825dd14113207113fabbd/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", size = 231230, upload-time = "2026-04-02T09:25:44.281Z" }, + { url = "https://files.pythonhosted.org/packages/20/f7/7b991776844dfa058017e600e6e55ff01984a063290ca5622c0b63162f68/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", size = 225890, upload-time = "2026-04-02T09:25:45.475Z" }, + { url = "https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", size = 216930, upload-time = "2026-04-02T09:25:46.58Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ab/b18f0ab31cdd7b3ddb8bb76c4a414aeb8160c9810fdf1bc62f269a539d87/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", size = 202109, upload-time = "2026-04-02T09:25:48.031Z" }, + { url = "https://files.pythonhosted.org/packages/82/e5/7e9440768a06dfb3075936490cb82dbf0ee20a133bf0dd8551fa096914ec/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", size = 214684, upload-time = "2026-04-02T09:25:49.245Z" }, + { url = "https://files.pythonhosted.org/packages/71/94/8c61d8da9f062fdf457c80acfa25060ec22bf1d34bbeaca4350f13bcfd07/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", size = 212785, upload-time = "2026-04-02T09:25:50.671Z" }, + { url = "https://files.pythonhosted.org/packages/66/cd/6e9889c648e72c0ab2e5967528bb83508f354d706637bc7097190c874e13/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", size = 203055, upload-time = "2026-04-02T09:25:51.802Z" }, + { url = "https://files.pythonhosted.org/packages/92/2e/7a951d6a08aefb7eb8e1b54cdfb580b1365afdd9dd484dc4bee9e5d8f258/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", size = 232502, upload-time = "2026-04-02T09:25:53.388Z" }, + { url = "https://files.pythonhosted.org/packages/58/d5/abcf2d83bf8e0a1286df55cd0dc1d49af0da4282aa77e986df343e7de124/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", size = 214295, upload-time = "2026-04-02T09:25:54.765Z" }, + { url = "https://files.pythonhosted.org/packages/47/3a/7d4cd7ed54be99973a0dc176032cba5cb1f258082c31fa6df35cff46acfc/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", size = 227145, upload-time = "2026-04-02T09:25:55.904Z" }, + { url = "https://files.pythonhosted.org/packages/1d/98/3a45bf8247889cf28262ebd3d0872edff11565b2a1e3064ccb132db3fbb0/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", size = 218884, upload-time = "2026-04-02T09:25:57.074Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/2e8b7f8915ed5c9ef13aa828d82738e33888c485b65ebf744d615040c7ea/charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", size = 148343, upload-time = "2026-04-02T09:25:58.199Z" }, + { url = "https://files.pythonhosted.org/packages/35/1b/3b8c8c77184af465ee9ad88b5aea46ea6b2e1f7b9dc9502891e37af21e30/charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", size = 159174, upload-time = "2026-04-02T09:25:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/feb40dca40dbb21e0a908801782d9288c64fc8d8e562c2098e9994c8c21b/charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", size = 147805, upload-time = "2026-04-02T09:26:00.756Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "sp-cli" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "click" }, + { name = "requests" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] From 8db09b124fc42620fcb911f3f9c79cc765e0706a Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 5 Aug 2026 23:21:18 +0530 Subject: [PATCH 02/10] feat: mirror the API's validators as Click choices Every enumeration the REST API validates is now restated in sp_cli/constants.py and wired into the commands as click.Choice, so a bad --status or --platform fails instantly as a usage error instead of costing a round trip and coming back as an HTTP 400. The values are not invented; each one mirrors a specific validator in the merged mod_api blueprint, and the module says which. The ones that are easy to guess wrong: - /runs?status= only accepts queued|running|canceled. It is derived from the latest TestProgress row, so pass/fail -- which are per-sample outcomes, not run states -- are deliberately absent. Use `sp run summary` for those. - _VALID_SAMPLE_STATUSES is pass|fail|missing_output|not_started. No 'skipped', no 'running'. - /regression-tests?active is a two-way switch, not tri-state: omitting it lists active tests only, so --active/--all was wrong and is now --active/--inactive. - There are seven token scopes, not six: system:write is separate from system:read so a monitoring token cannot reconfigure the platform. Also fills in filters the API supports but the CLI was not exposing (sample --sha256/--status, queue --limit/--offset, run --repository/--sort and the date window). ApiContractTests pins all of this, so a validator changing upstream fails here loudly rather than in production. --- sp_cli/commands/auth.py | 59 ++++++++-- sp_cli/commands/regression.py | 10 +- sp_cli/commands/run.py | 124 +++++++++++++++++++-- sp_cli/commands/sample.py | 40 +++++-- sp_cli/commands/system.py | 21 +++- sp_cli/constants.py | 109 +++++++++++++++++++ tests/test_cli.py | 196 ++++++++++++++++++++++++++++++++++ 7 files changed, 527 insertions(+), 32 deletions(-) create mode 100644 sp_cli/constants.py diff --git a/sp_cli/commands/auth.py b/sp_cli/commands/auth.py index 3de56a8..0b38223 100644 --- a/sp_cli/commands/auth.py +++ b/sp_cli/commands/auth.py @@ -1,38 +1,81 @@ -"""``sp auth`` — obtain and revoke API tokens.""" +"""``sp auth`` — obtain, list, and revoke API tokens.""" + +from typing import Optional, Tuple import click from sp_cli.client import ApiError +from sp_cli.constants import TOKEN_MAX_DAYS, TOKEN_MIN_DAYS, TOKEN_SCOPES from sp_cli.output import render, render_error +from sp_cli.runner import clean_params, fetch_and_render @click.group() def auth() -> None: - """Obtain and revoke API tokens.""" + """Obtain, list, and revoke API tokens.""" @auth.command('login') @click.option('--email', prompt=True, help='Account email.') @click.option('--password', prompt=True, hide_input=True, help='Account password (never stored).') -@click.option('--name', 'token_name', default='sp-cli', show_default=True, help='Token label.') -@click.option('--days', 'expires_in_days', type=int, default=30, show_default=True, - help='Token lifetime in days (max 90).') +@click.option('--name', 'token_name', default='sp-cli', show_default=True, + help='Token label; must match ^[a-zA-Z0-9_-]+$.') +@click.option('--days', 'expires_in_days', type=click.IntRange(TOKEN_MIN_DAYS, TOKEN_MAX_DAYS), + default=TOKEN_MAX_DAYS, show_default=True, help='Token lifetime in days.') +@click.option('--scope', 'scopes', multiple=True, type=click.Choice(TOKEN_SCOPES), + help='Grant a specific scope; repeatable. Omit for the server default set.') @click.pass_context -def auth_login(ctx: click.Context, email: str, password: str, - token_name: str, expires_in_days: int) -> None: - """Create an API token; store the printed value in SP_API_TOKEN.""" +def auth_login(ctx: click.Context, email: str, password: str, token_name: str, + expires_in_days: int, scopes: Tuple[str, ...]) -> None: + """Create an API token; store the printed value in SP_API_TOKEN. + + The plaintext token is returned exactly once, at creation. Later `sp auth + tokens` calls list metadata only, so capture it now or create a new one. + """ client = ctx.obj['client'] output = ctx.obj['output'] body = {'email': email, 'password': password, 'token_name': token_name, 'expires_in_days': expires_in_days} + if scopes: + body['scopes'] = list(scopes) try: result = client.request('POST', '/auth/tokens', json_body=body) except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) + render(result, output) +@auth.command('tokens') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def auth_tokens(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: + """List this account's API tokens (metadata only, never the token itself).""" + params = clean_params({'limit': limit, 'offset': offset}) + fetch_and_render(ctx, '/auth/tokens', params) + + +@auth.command('revoke') +@click.argument('token_id', type=int) +@click.pass_context +def auth_revoke(ctx: click.Context, token_id: int) -> None: + """Revoke one token by id, leaving the token in use untouched. + + Use `sp auth tokens` to find the id. To revoke the token you are currently + authenticating with, use `sp auth logout` instead. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + client.request('DELETE', f'/auth/tokens/{token_id}') + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + click.echo(f'Token {token_id} revoked.') + + @auth.command('logout') @click.pass_context def auth_logout(ctx: click.Context) -> None: diff --git a/sp_cli/commands/regression.py b/sp_cli/commands/regression.py index fac9e37..ec006d7 100644 --- a/sp_cli/commands/regression.py +++ b/sp_cli/commands/regression.py @@ -15,7 +15,8 @@ def regression() -> None: @regression.command('ls') @click.option('--category', default=None, help='Filter by category name.') @click.option('--tag', default=None, help='Filter by tag.') -@click.option('--active/--all', 'active', default=None, help='Only active tests (default: all).') +@click.option('--active/--inactive', 'active', default=None, + help='Select active or inactive tests (default: active only).') @click.option('--sample-id', type=int, default=None, help='Filter by sample id.') @click.option('--limit', type=int, default=None, help='Page size (max 100).') @click.option('--offset', type=int, default=None, help='Pagination offset.') @@ -23,7 +24,12 @@ def regression() -> None: def regression_ls(ctx: click.Context, category: Optional[str], tag: Optional[str], active: Optional[bool], sample_id: Optional[int], limit: Optional[int], offset: Optional[int]) -> None: - """List regression-test definitions.""" + """List regression-test definitions. + + The API's ``active`` filter is a two-way switch with no "everything" + setting: omitting it lists active tests only, and ``--inactive`` lists + inactive ones only. Run the command twice to see both. + """ params = clean_params({'category': category, 'tag': tag, 'active': active, 'sample_id': sample_id, 'limit': limit, 'offset': offset}) fetch_and_render(ctx, '/regression-tests', params) diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index b304248..6f5ec54 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -5,6 +5,8 @@ import click from sp_cli.client import ApiError +from sp_cli.constants import (CANCEL_REASON_MIN_LENGTH, PLATFORMS, + RUN_STATUSES, SAMPLE_STATUSES) from sp_cli.output import render, render_error from sp_cli.runner import clean_params, fetch_and_render from sp_cli.triage import classify_sample, is_failure @@ -16,18 +18,34 @@ def run() -> None: @run.command('ls') -@click.option('--status', default=None, help='queued|running|pass|fail|canceled|error|incomplete') -@click.option('--platform', default=None, help='linux|windows') +@click.option('--status', type=click.Choice(RUN_STATUSES), default=None, + help='Filter by lifecycle status. The API only tracks these three.') +@click.option('--platform', type=click.Choice(PLATFORMS), default=None, help='Test platform.') @click.option('--branch', default=None, help='Filter by branch name.') @click.option('--commit', 'commit_sha', default=None, help='Full 40-char commit SHA.') +@click.option('--repository', default=None, help='Filter by fork, as owner/repo.') +@click.option('--sort', default=None, help='Sort key, e.g. -created_at (default) or run_id.') +@click.option('--created-after', 'created_after', default=None, + help='Only runs first seen at/after this time (ISO 8601).') +@click.option('--created-before', 'created_before', default=None, + help='Only runs first seen at/before this time (ISO 8601).') @click.option('--limit', type=int, default=None, help='Page size (max 100).') @click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context def run_ls(ctx: click.Context, status: Optional[str], platform: Optional[str], branch: Optional[str], - commit_sha: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: - """List CI runs (newest first).""" + commit_sha: Optional[str], repository: Optional[str], sort: Optional[str], + created_after: Optional[str], created_before: Optional[str], + limit: Optional[int], offset: Optional[int]) -> None: + """List CI runs (newest first). + + --status only accepts queued, running, and canceled: the API derives them + from the latest TestProgress row, and pass/fail are per-sample outcomes + rather than run states. Use `sp run summary` for a run's pass/fail split. + """ params = clean_params({'status': status, 'platform': platform, 'branch': branch, - 'commit_sha': commit_sha, 'limit': limit, 'offset': offset}) + 'commit_sha': commit_sha, 'repository': repository, 'sort': sort, + 'created_after': created_after, 'created_before': created_before, + 'limit': limit, 'offset': offset}) fetch_and_render(ctx, '/runs', params) @@ -66,14 +84,20 @@ def run_failures(ctx: click.Context, run_id: int) -> None: @run.command('results') @click.argument('run_id', type=int) -@click.option('--status', default=None, help='pass|fail|skipped|missing_output|running|not_started') +@click.option('--status', type=click.Choice(SAMPLE_STATUSES), default=None, + help='Filter by per-sample outcome.') +@click.option('--name', default=None, help='Substring match on the sample name.') +@click.option('--tag', default=None, help='Filter by sample tag.') +@click.option('--category', default=None, help='Filter by regression-test category.') @click.option('--limit', type=int, default=None, help='Page size (max 100).') @click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context -def run_results(ctx: click.Context, run_id: int, status: Optional[str], +def run_results(ctx: click.Context, run_id: int, status: Optional[str], name: Optional[str], + tag: Optional[str], category: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: """List all regression-test results in a run.""" - params = clean_params({'status': status, 'limit': limit, 'offset': offset}) + params = clean_params({'status': status, 'name': name, 'tag': tag, 'category': category, + 'limit': limit, 'offset': offset}) fetch_and_render(ctx, f'/runs/{run_id}/samples', params) @@ -167,6 +191,90 @@ def run_approve_baseline(ctx: click.Context, run_id: int, sample_id: int, render(result, output) +@run.command('progress') +@click.argument('run_id', type=int) +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def run_progress(ctx: click.Context, run_id: int, limit: Optional[int], + offset: Optional[int]) -> None: + """Show the timeline of progress events the CI worker recorded for a run.""" + params = clean_params({'limit': limit, 'offset': offset}) + fetch_and_render(ctx, f'/runs/{run_id}/progress', params) + + +@run.command('config') +@click.argument('run_id', type=int) +@click.pass_context +def run_config(ctx: click.Context, run_id: int) -> None: + """Show the platform, branch, commit, and regression tests a run was launched with.""" + fetch_and_render(ctx, f'/runs/{run_id}/config') + + +@run.command('cancel') +@click.argument('run_id', type=int) +@click.option('--reason', default=None, + help=f'Why the run is being canceled (min {CANCEL_REASON_MIN_LENGTH} characters).') +@click.pass_context +def run_cancel(ctx: click.Context, run_id: int, reason: Optional[str]) -> None: + """Cancel a queued or running test. + + Requires the ``runs:write`` scope. Idempotent: canceling a run that has + already finished succeeds and reports ``status=no_op`` rather than failing. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + if reason is not None and len(reason.strip()) < CANCEL_REASON_MIN_LENGTH: + raise click.BadParameter( + f'must be at least {CANCEL_REASON_MIN_LENGTH} characters', param_hint='--reason') + body = {'reason': reason} if reason else None + try: + result = client.request('POST', f'/runs/{run_id}/cancel', json_body=body) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + render(result, output) + + +@run.command('output') +@click.argument('run_id', type=int) +@click.argument('sample_id', type=int) +@click.option('--regression', 'regression_id', type=int, default=None, + help='Regression test id (auto-resolved if omitted).') +@click.option('--output', 'output_id', type=int, default=None, + help='Output file id (auto-resolved if omitted).') +@click.option('--side', type=click.Choice(('expected', 'actual')), default='actual', + show_default=True, help='Which side of the comparison to fetch.') +@click.option('--format', 'fmt', default=None, help='Response format accepted by the API.') +@click.pass_context +def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: Optional[int], + output_id: Optional[int], side: str, fmt: Optional[str]) -> None: + """Fetch one side of a result's output file. + + Resolves the (media sample, regression, output) ids the same way `sp run + diff` does, so the hidden ids the web UI needs are not required here. + + Note: for an output that matched, the API answers ``actual`` with a 303 + redirect to ``expected`` -- requests follows it, so the expected content is + what comes back. + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + targets = _resolve_diff_targets(client, run_id, sample_id, regression_id, output_id) + if not targets: + raise ApiError('not_found', 'No output to fetch for this result', 404) + media_sample_id, reg_id, out_id = targets[0] + payload = client.get( + f'/runs/{run_id}/samples/{media_sample_id}' + f'/regression-tests/{reg_id}/outputs/{out_id}/{side}', + params=clean_params({'format': fmt})) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + render(payload, output) + + @run.command('artifacts') @click.argument('run_id', type=int) @click.pass_context diff --git a/sp_cli/commands/sample.py b/sp_cli/commands/sample.py index d8bd8ae..cca7b1a 100644 --- a/sp_cli/commands/sample.py +++ b/sp_cli/commands/sample.py @@ -4,6 +4,8 @@ import click +from sp_cli.constants import (PLATFORMS, SAMPLE_CATALOG_STATUSES, + SAMPLE_STATUSES) from sp_cli.runner import clean_params, fetch_and_render @@ -13,17 +15,21 @@ def sample() -> None: @sample.command('ls') -@click.option('--name', default=None, help='Filter by sample name.') +@click.option('--name', default=None, help='Substring match on the original sample name.') @click.option('--tag', default=None, help='Filter by tag.') @click.option('--extension', default=None, help='Filter by file extension.') +@click.option('--sha256', default=None, help='Filter by exact SHA-256 hash.') +@click.option('--status', type=click.Choice(SAMPLE_CATALOG_STATUSES), default=None, + help='Catalog visibility, not a test outcome.') @click.option('--limit', type=int, default=None, help='Page size (max 100).') @click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context -def sample_ls(ctx: click.Context, name: Optional[str], tag: Optional[str], - extension: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: +def sample_ls(ctx: click.Context, name: Optional[str], tag: Optional[str], extension: Optional[str], + sha256: Optional[str], status: Optional[str], + limit: Optional[int], offset: Optional[int]) -> None: """List known media samples.""" - params = clean_params({'name': name, 'tag': tag, 'extension': extension, - 'limit': limit, 'offset': offset}) + params = clean_params({'name': name, 'tag': tag, 'extension': extension, 'sha256': sha256, + 'status': status, 'limit': limit, 'offset': offset}) fetch_and_render(ctx, '/samples', params) @@ -37,11 +43,27 @@ def sample_show(ctx: click.Context, sample_id: int) -> None: @sample.command('history') @click.argument('sample_id', type=int) -@click.option('--platform', default=None, help='linux|windows') +@click.option('--platform', type=click.Choice(PLATFORMS), default=None, help='Test platform.') +@click.option('--branch', default=None, help='Filter by branch name.') +@click.option('--status', type=click.Choice(SAMPLE_STATUSES), default=None, + help='Filter by this sample\'s outcome in each run.') +@click.option('--created-after', 'created_after', default=None, + help='Only runs first seen at/after this time (ISO 8601).') +@click.option('--created-before', 'created_before', default=None, + help='Only runs first seen at/before this time (ISO 8601).') @click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context def sample_history(ctx: click.Context, sample_id: int, platform: Optional[str], - limit: Optional[int]) -> None: - """Show this sample's result history across runs.""" - params = clean_params({'platform': platform, 'limit': limit}) + branch: Optional[str], status: Optional[str], created_after: Optional[str], + created_before: Optional[str], limit: Optional[int], + offset: Optional[int]) -> None: + """Show this sample's result history across runs. + + Each entry carries a ``failure_signature``, which is what separates a real + regression from an infra flake that happens to fail the same test. + """ + params = clean_params({'platform': platform, 'branch': branch, 'status': status, + 'created_after': created_after, 'created_before': created_before, + 'limit': limit, 'offset': offset}) fetch_and_render(ctx, f'/samples/{sample_id}/history', params) diff --git a/sp_cli/commands/system.py b/sp_cli/commands/system.py index cb913a2..fcca38a 100644 --- a/sp_cli/commands/system.py +++ b/sp_cli/commands/system.py @@ -4,6 +4,7 @@ import click +from sp_cli.constants import PLATFORMS, QUEUE_STATUSES from sp_cli.runner import clean_params, fetch_and_render @@ -15,9 +16,19 @@ def health(ctx: click.Context) -> None: @click.command('queue') -@click.option('--platform', default=None, help='linux|windows') -@click.option('--status', default=None, help='queued|running') +@click.option('--platform', type=click.Choice(PLATFORMS), default=None, help='Test platform.') +@click.option('--status', type=click.Choice(QUEUE_STATUSES), default=None, + help='Restrict to one side of the queue.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context -def queue(ctx: click.Context, platform: Optional[str], status: Optional[str]) -> None: - """Show queue depth and currently running jobs.""" - fetch_and_render(ctx, '/system/queue', clean_params({'platform': platform, 'status': status})) +def queue(ctx: click.Context, platform: Optional[str], status: Optional[str], + limit: Optional[int], offset: Optional[int]) -> None: + """Show queue depth and currently running jobs. + + Completed and canceled runs are excluded. The per-item ``position`` field + is only populated when ``--status queued`` is passed; otherwise it is null. + """ + params = clean_params({'platform': platform, 'status': status, + 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, '/system/queue', params) diff --git a/sp_cli/constants.py b/sp_cli/constants.py new file mode 100644 index 0000000..72f7646 --- /dev/null +++ b/sp_cli/constants.py @@ -0,0 +1,109 @@ +"""Enumerations accepted by the platform API, mirrored so the CLI can reject bad input locally. + +Each tuple matches a validator in the merged ``mod_api`` blueprint. Keeping them +here means a wrong ``--status`` fails instantly with a Click usage error instead +of costing a round trip and coming back as an HTTP 400. +""" + +#: ``GET /runs`` — derived from the latest ``TestProgress`` row. pass/fail are +#: per-sample outcomes, not run states, so they are deliberately absent. +RUN_STATUSES = ('queued', 'running', 'canceled') + +#: ``GET /runs/{id}/samples`` and ``GET /samples/{id}/history`` (``_VALID_SAMPLE_STATUSES``). +SAMPLE_STATUSES = ('pass', 'fail', 'missing_output', 'not_started') + +#: ``GET /system/queue`` — a queued run is one with no running progress row yet. +QUEUE_STATUSES = ('queued', 'running') + +#: ``GET /samples`` — catalog visibility, unrelated to test outcome. +SAMPLE_CATALOG_STATUSES = ('active', 'inactive') + +#: ``TestPlatform`` values accepted by every ``?platform`` filter. +PLATFORMS = ('linux', 'windows') + +#: ``TokenCreateRequestSchema.expires_in_days`` validates ``Range(min=1, max=30)``. +TOKEN_MIN_DAYS = 1 +TOKEN_MAX_DAYS = 30 + +#: ``mod_api.models.api_token.VALID_SCOPES``. Omitting scopes at login grants the +#: server's default set (``runs:read`` + ``results:read``), which is why +#: ``--scope`` is optional. ``system:write`` is deliberately separate from +#: ``system:read`` so a monitoring token can watch the platform without being +#: able to reconfigure it — every ``sp admin`` write command needs it. +TOKEN_SCOPES = ('runs:read', 'runs:write', 'results:read', 'baselines:write', + 'system:read', 'system:write', 'tokens:manage') + +#: ``TokenCreateRequestSchema.scopes`` validates ``Length(max=len(VALID_SCOPES))``, +#: so asking for everything you are allowed can never fail validation. +TOKEN_MAX_SCOPES = len(TOKEN_SCOPES) + +#: ``POST /runs/{id}/cancel`` rejects a reason shorter than this. +CANCEL_REASON_MIN_LENGTH = 5 + +#: ``GET /runs/{id}/errors`` — the types ``derive_errors_for_run`` can emit. +#: Test errors are derived from result rows, not stored, so this is the closed set. +ERROR_TYPES = ('exit_code_mismatch', 'missing_output', 'diff_mismatch') + +#: ``GET /runs/{id}/infrastructure-errors`` — ``_classify_infra_error`` keyword buckets. +INFRA_ERROR_TYPES = ('vm_provisioning', 'checkout', 'merge', 'build', 'worker', 'storage') + +#: ``mod_api.services.error_service._SEVERITY_ORDER``. Test errors are only ever +#: ``error`` or ``warning``; infrastructure errors are always ``critical``. +ERROR_SEVERITIES = ('info', 'warning', 'error', 'critical') + +#: ``GET /runs/{id}/error-summary`` — anything else is a 400. +ERROR_GROUP_BY = ('type', 'severity', 'sample_id', 'regression_id') + +#: ``GET /runs/{id}/artifacts`` — the ``?type`` values ``list_artifacts`` builds. +ARTIFACT_TYPES = ('binary', 'coredump', 'combined_stdout', 'build_log', + 'expected_output', 'actual_output') + +#: ``GET /runs/{id}/logs`` — ``_extract_level`` scans raw lines for these, so a +#: line matching none of them is reported as ``info``. +LOG_LEVELS = ('critical', 'error', 'warning', 'info', 'debug') + +#: ``GET /runs/{id}/logs`` — ``_extract_source`` keywords; unmatched lines are ``web``. +LOG_SOURCES = ('orchestrator', 'worker', 'build', 'test_runner', 'web') + +#: ``read_log_lines`` clamps the page size into this range server-side. +LOG_MAX_LIMIT = 500 + +#: ``GET /runs/{id}/logs`` rejects a longer ``contains`` filter with a 400. +LOG_CONTAINS_MAX_LENGTH = 100 + +#: ``mod_regression.models.InputType`` — ``RegressionTestCreateSchema.input_type``. +INPUT_TYPES = ('file', 'stdin', 'udp') + +#: ``mod_regression.models.OutputType``. Note ``multi_program`` serializes as +#: ``multiprogram``: the enum's *value* is what the API validates against. +OUTPUT_TYPES = ('file', 'null', 'tcp', 'cea708', 'multiprogram', 'stdout', 'report') + +#: ``mod_auth.models.Role`` — accepted by ``PATCH /users/{id}``. +USER_ROLES = ('admin', 'contributor', 'tester', 'user') + +#: ``RegressionTestCreateSchema.command`` — ``Length(min=1, max=500)``. +COMMAND_MAX_LENGTH = 500 + +#: Category column widths, shared by the regression-test and category schemas. +NAME_MAX_LENGTH = 64 +DESCRIPTION_MAX_LENGTH = 1024 + +#: ``expected_rc`` is a process exit code — ``Range(min=0, max=255)``. +EXPECTED_RC_MIN = 0 +EXPECTED_RC_MAX = 255 + +#: ``POST /runs`` — ``commit_sha`` must be a full 40-char hex string, no short SHAs. +COMMIT_SHA_LENGTH = 40 + +#: ``POST /runs`` — ``regression_test_ids`` is capped at ``Length(max=500)``. +MAX_REGRESSION_TEST_IDS = 500 + +#: ``POST /runs`` — ``branch`` and ``repository`` widths. +BRANCH_MAX_LENGTH = 100 +REPOSITORY_MAX_LENGTH = 100 + +#: ``ForbiddenExtensionCreateSchema`` — alphanumeric, no leading dot, stored lower-cased. +EXTENSION_MAX_LENGTH = 32 + +#: ``BlockedUserCreateSchema.comment`` — ``Length(max=1024)``. +BLOCKED_USER_COMMENT_MAX_LENGTH = 1024 diff --git a/tests/test_cli.py b/tests/test_cli.py index 4836c57..6e898ef 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -26,6 +26,15 @@ 'status': 'pass', 'exit_code': 0, 'expected_rc': 0, 'outputs': []}, ] +# Two failures on distinct media samples, which is what history lookups key on. +SAMPLES_WITH_IDS = [ + {'regression_test_id': 18, 'sample_id': 42, 'sample_name': 'dvb', 'categories': ['DVB'], + 'status': 'fail', 'exit_code': 10, 'expected_rc': 0, 'outputs': []}, + {'regression_test_id': 137, 'sample_id': 43, 'sample_name': 'cea708', + 'categories': ['CEA-708'], 'status': 'missing_output', 'exit_code': 0, + 'expected_rc': 0, 'outputs': []}, +] + class CliCommandTests(unittest.TestCase): """Exercise the CLI commands with a mocked client.""" @@ -184,3 +193,190 @@ def test_investigate_combines_run_summary_and_failures(self, mock_get, mock_pagi self.assertEqual(report['by_code'], {'SEGFAULT': 1, 'EXIT_CODE_MISMATCH': 1, 'MISSING_OUTPUT': 1}) self.assertEqual(len(report['failures']), 3) + + +class ApiContractTests(unittest.TestCase): + """Pin the CLI surface to what the merged ``mod_api`` blueprint actually accepts. + + Every enumeration asserted here mirrors a validator in the platform repo, so + a drift on either side should fail loudly instead of costing an HTTP 400. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_ls_rejects_statuses_the_api_does_not_support(self, mock_get): + """`run ls --status` only offers queued/running/canceled, and rejects the rest locally.""" + for bad_status in ('pass', 'fail', 'error', 'incomplete'): + result = self.runner.invoke(cli, ['run', 'ls', '--status', bad_status]) + self.assertNotEqual(result.exit_code, 0, f'{bad_status} should be rejected') + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_ls_forwards_repository_sort_and_date_filters(self, mock_get): + """`run ls` passes through every filter /runs declares.""" + mock_get.return_value = RUNS_PAGE + result = self.runner.invoke(cli, [ + 'run', 'ls', '--repository', 'CCExtractor/ccextractor', '--sort', '-created_at', + '--created-after', '2026-07-01T00:00:00Z', '--created-before', '2026-07-31T00:00:00Z']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs', params={ + 'repository': 'CCExtractor/ccextractor', 'sort': '-created_at', + 'created_after': '2026-07-01T00:00:00Z', 'created_before': '2026-07-31T00:00:00Z'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_results_rejects_unsupported_status(self, mock_get): + """'skipped' and 'running' are not in the API's _VALID_SAMPLE_STATUSES.""" + for bad_status in ('skipped', 'running'): + result = self.runner.invoke(cli, ['run', 'results', '9299', '--status', bad_status]) + self.assertNotEqual(result.exit_code, 0, f'{bad_status} should be rejected') + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_results_forwards_name_tag_and_category(self, mock_get): + """`run results` supports the joined-field filters /runs/{id}/samples applies.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['run', 'results', '9299', '--status', 'missing_output', + '--name', 'dvb', '--tag', 'teletext', '--category', 'DVB']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/samples', params={ + 'status': 'missing_output', 'name': 'dvb', 'tag': 'teletext', 'category': 'DVB'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_progress_and_config(self, mock_get): + """`run progress` and `run config` reach their merged endpoints.""" + mock_get.return_value = {'data': [], 'pagination': {}} + self.assertEqual(self.runner.invoke(cli, ['run', 'progress', '9299']).exit_code, 0) + mock_get.assert_called_with('/runs/9299/progress', params={}) + + mock_get.return_value = {'run_id': 9299, 'platform': 'windows'} + self.assertEqual(self.runner.invoke(cli, ['run', 'config', '9299']).exit_code, 0) + mock_get.assert_called_with('/runs/9299/config', params=None) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_run_cancel_posts_with_reason(self, mock_request): + """`run cancel` POSTs the reason when one is given.""" + mock_request.return_value = {'run_id': 9299, 'action': 'cancel', 'status': 'canceled'} + result = self.runner.invoke(cli, ['run', 'cancel', '9299', '--reason', 'superseded by 9300']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with('POST', '/runs/9299/cancel', + json_body={'reason': 'superseded by 9300'}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_run_cancel_omits_body_when_no_reason(self, mock_request): + """No reason means no body, rather than a null the schema would reject.""" + mock_request.return_value = {'run_id': 9299, 'status': 'canceled'} + result = self.runner.invoke(cli, ['run', 'cancel', '9299']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with('POST', '/runs/9299/cancel', json_body=None) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_run_cancel_rejects_short_reason_before_the_request(self, mock_request): + """The API needs 5+ characters; catch it locally instead of round-tripping a 400.""" + result = self.runner.invoke(cli, ['run', 'cancel', '9299', '--reason', 'no']) + + self.assertNotEqual(result.exit_code, 0) + mock_request.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_output_resolves_ids_like_diff(self, mock_get): + """`run output` reuses the diff resolver, so the hidden ids stay optional.""" + mock_get.side_effect = [ + {'regression_test_id': 137, 'sample_id': 42, + 'outputs': [{'output_id': 2, 'status': 'fail'}]}, + {'content': 'line', 'truncated': False}, + ] + result = self.runner.invoke(cli, ['run', 'output', '9299', '5', '--side', 'expected']) + + self.assertEqual(result.exit_code, 0) + args, kwargs = mock_get.call_args + self.assertEqual(args[0], '/runs/9299/samples/42/regression-tests/137/outputs/2/expected') + self.assertEqual(kwargs['params'], {}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_auth_login_rejects_lifetime_over_the_api_cap(self, mock_request): + """expires_in_days is validated as Range(min=1, max=30) server-side.""" + result = self.runner.invoke(cli, ['auth', 'login', '--email', 'a@b.co', + '--password', 'hunter22', '--days', '60']) + + self.assertNotEqual(result.exit_code, 0) + mock_request.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_auth_login_sends_scopes_only_when_requested(self, mock_request): + """Omitting --scope leaves the field out so the server picks its default set.""" + mock_request.return_value = {'token': 'spci_x', 'token_name': 'sp-cli', 'scopes': []} + result = self.runner.invoke(cli, ['auth', 'login', '--email', 'a@b.co', + '--password', 'hunter22', '--scope', 'runs:read', + '--scope', 'results:read']) + + self.assertEqual(result.exit_code, 0) + body = mock_request.call_args.kwargs['json_body'] + self.assertEqual(body['scopes'], ['runs:read', 'results:read']) + self.assertEqual(body['expires_in_days'], 30) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_auth_tokens_lists_metadata(self, mock_get): + """`auth tokens` reads the list endpoint.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['auth', 'tokens']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/auth/tokens', params={}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_auth_revoke_targets_one_token(self, mock_request): + """`auth revoke ` deletes that token, not the current one.""" + mock_request.return_value = None + result = self.runner.invoke(cli, ['auth', 'revoke', '7']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with('DELETE', '/auth/tokens/7') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_sample_ls_forwards_sha256_and_catalog_status(self, mock_get): + """`sample ls` covers every filter /samples declares.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['sample', 'ls', '--sha256', 'abc123', + '--status', 'inactive']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/samples', + params={'sha256': 'abc123', 'status': 'inactive'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_sample_history_forwards_branch_and_date_window(self, mock_get): + """`sample history` supports the branch/date filters the endpoint applies.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['sample', 'history', '42', '--branch', 'master', + '--status', 'fail', '--created-after', '2026-07-01T00:00:00Z']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/samples/42/history', params={ + 'branch': 'master', 'status': 'fail', 'created_after': '2026-07-01T00:00:00Z'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_regression_ls_inactive_is_a_two_way_switch(self, mock_get): + """--inactive sends active=false; omitting it lets the API default to active only.""" + mock_get.return_value = {'data': [], 'pagination': {}} + self.assertEqual(self.runner.invoke(cli, ['regression', 'ls', '--inactive']).exit_code, 0) + mock_get.assert_called_with('/regression-tests', params={'active': False}) + + self.assertEqual(self.runner.invoke(cli, ['regression', 'ls']).exit_code, 0) + mock_get.assert_called_with('/regression-tests', params={}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_queue_rejects_non_queue_status_and_paginates(self, mock_get): + """/system/queue only knows queued and running, and accepts pagination.""" + self.assertNotEqual(self.runner.invoke(cli, ['queue', '--status', 'canceled']).exit_code, 0) + + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['queue', '--status', 'queued', '--limit', '10']) + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/system/queue', params={'status': 'queued', 'limit': 10}) From add6715efa59b1840f1900ce7c4eadaccc16396c Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 5 Aug 2026 23:25:10 +0530 Subject: [PATCH 03/10] feat: sp investigate --with-history The classification codes say what broke, but not whether it is new. This adds a cross-run verdict to every failure so the first question after a red build -- "did I break this?" -- is answered without opening the web UI. Verdicts: NEW_REGRESSION (passed in the previous run, start reading here), STILL_FAILING, NEVER_PASSED, FLAKY (two or more pass/fail flips), NO_HISTORY, UNKNOWN. --history-depth N implies the flag. The filtering in history.split_history is the part worth reviewing: the /samples/{id}/history endpoint returns every regression test for a sample and includes the run being investigated, so both have to be stripped before any verdict is inferred. Leaving the current run in makes everything look STILL_FAILING; leaving sibling tests in makes unrelated failures look like history for this one. Costs one lookup per distinct media sample, filtered to the run's platform and cached across regression tests that share a sample -- so a 200-test run with 30 distinct failing samples is 30 calls, not 200. --- sp_cli/commands/investigate.py | 136 +++++++++++++++++++++++-- sp_cli/history.py | 175 +++++++++++++++++++++++++++++++++ tests/test_cli.py | 160 ++++++++++++++++++++++++++++++ tests/test_history.py | 165 +++++++++++++++++++++++++++++++ 4 files changed, 629 insertions(+), 7 deletions(-) create mode 100644 sp_cli/history.py create mode 100644 tests/test_history.py diff --git a/sp_cli/commands/investigate.py b/sp_cli/commands/investigate.py index 16c2a24..07594ce 100644 --- a/sp_cli/commands/investigate.py +++ b/sp_cli/commands/investigate.py @@ -1,27 +1,49 @@ """``sp investigate`` — one-shot triage of a run (status + counts + classified failures).""" -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import click from sp_cli.client import ApiError +from sp_cli.history import (NEW_REGRESSION, classify_history, group_by_verdict, + split_history, unknown_history) from sp_cli.output import render, render_error +from sp_cli.runner import clean_params from sp_cli.triage import classify_sample, group_by_code, is_failure _RUN_FIELDS = ('run_id', 'pr_number', 'platform', 'commit_sha', 'branch', 'status', 'github_link') +#: How many prior runs to weigh per failure when --with-history is used. Deep +#: enough to see a sample settle, shallow enough to keep it one call per sample. +DEFAULT_HISTORY_DEPTH = 20 + @click.command('investigate') @click.argument('run_id', type=int) +@click.option('--with-history', 'with_history', is_flag=True, default=False, + help='Label each failure as a new regression, long-standing, or never-passing.') +@click.option('--history-depth', type=int, default=None, + help=f'Prior runs to weigh per failure (default: {DEFAULT_HISTORY_DEPTH}). ' + 'Implies --with-history.') @click.pass_context -def investigate(ctx: click.Context, run_id: int) -> None: +def investigate(ctx: click.Context, run_id: int, with_history: bool, + history_depth: Optional[int]) -> None: """Triage a run in one shot: run info, pass/fail counts, and classified failures. Combines the run detail, summary, and per-result classification into a single - report — the whole "what failed and why" investigation in one command. + report -- the whole "what failed and why" investigation in one command. + + --with-history answers the question the codes alone cannot: is this failure + new? It adds a `history` block to every failure plus a `by_verdict` tally, + at the cost of one extra API call per distinct sample. NEW_REGRESSION means + the test passed in the previous run, which is where to start reading. """ client = ctx.obj['client'] output = ctx.obj['output'] + if history_depth is not None: + with_history = True + depth = history_depth if history_depth is not None else DEFAULT_HISTORY_DEPTH + try: run = client.get(f'/runs/{run_id}') summary = client.get(f'/runs/{run_id}/summary') @@ -31,25 +53,74 @@ def investigate(ctx: click.Context, run_id: int) -> None: raise SystemExit(error.exit_code) failures = [classify_sample(s) for s in samples if is_failure(s)] - report = { + report: Dict[str, Any] = { 'run': {field: run.get(field) for field in _RUN_FIELDS}, 'summary': summary, 'by_code': group_by_code(failures), 'failures': failures, } + if with_history: + try: + _attach_history(client, failures, run_id, run.get('platform'), depth) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + report['by_verdict'] = group_by_verdict(failures) + if output == 'json': render(report, 'json') else: - _print_digest(report) + _print_digest(report, with_history) + + +def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, + platform: Optional[str], depth: int) -> None: + """ + Add a ``history`` verdict block to every failure row, in place. + History is fetched per *sample*, but several regression tests can share one + sample, so responses are cached by sample id and then narrowed per failure + by regression test id. Restricting to the run's own platform keeps a Windows + failure from being judged against Linux history. -def _print_digest(report: Dict[str, Any]) -> None: + :param client: The API client. + :type client: Any + :param failures: Classified failure rows, mutated in place. + :type failures: List[Dict[str, Any]] + :param run_id: The run being investigated. + :type run_id: int + :param platform: The run's platform, used to filter history. + :type platform: Optional[str] + :param depth: How many prior runs to consider per failure. + :type depth: int + """ + cache: Dict[int, List[Dict[str, Any]]] = {} + for failure in failures: + sample_id = failure.get('sample_id') + if not isinstance(sample_id, int): + failure['history'] = unknown_history('Result has no sample id to look up') + continue + + if sample_id not in cache: + # +1 so the current run's own entry cannot displace an older one. + params = clean_params({'platform': platform, 'limit': depth + 1}) + cache[sample_id] = client.get_paginated( + f'/samples/{sample_id}/history', params=params, max_items=depth + 1) + + current, prior = split_history(cache[sample_id], run_id, + failure.get('regression_test_id')) + failure['history'] = classify_history(current, prior[:depth]) + + +def _print_digest(report: Dict[str, Any], with_history: bool = False) -> None: """ Print a human-readable investigation digest. :param report: The assembled investigation report. :type report: Dict[str, Any] + :param with_history: Whether history verdicts were collected. + :type with_history: bool """ run = report['run'] summary = report['summary'] @@ -66,7 +137,58 @@ def _print_digest(report: Dict[str, Any]) -> None: for code, count in by_code.items(): click.echo(f" {str(count).rjust(4)} {code}") + by_verdict = report.get('by_verdict') + if by_verdict: + click.echo() + click.echo(" by history:") + for verdict, count in by_verdict.items(): + click.echo(f" {str(count).rjust(4)} {verdict}") + failures: List[Dict[str, Any]] = report['failures'] if failures: click.echo() - render({'data': failures}, 'table') + render({'data': [_flatten(f, with_history) for f in failures]}, 'table') + + _print_regressions(failures, with_history) + + +def _print_regressions(failures: List[Dict[str, Any]], with_history: bool) -> None: + """ + Call out the failures that were passing in the previous run. + + :param failures: Classified failure rows. + :type failures: List[Dict[str, Any]] + :param with_history: Whether history verdicts were collected. + :type with_history: bool + """ + if not with_history: + return + regressions = [f for f in failures + if f.get('history', {}).get('verdict') == NEW_REGRESSION] + if not regressions: + return + click.echo() + click.echo(f" {len(regressions)} of these were passing in the previous run:") + for failure in regressions: + click.echo(f" {failure.get('sample_name')} — {failure['history']['reason']}") + + +def _flatten(failure: Dict[str, Any], with_history: bool) -> Dict[str, Any]: + """ + Flatten a failure row for the table view, lifting the verdict into a column. + + The nested ``history`` block is right for JSON and wrong for a table, which + needs scalar cells. + + :param failure: One classified failure row. + :type failure: Dict[str, Any] + :param with_history: Whether to add the verdict column. + :type with_history: bool + :return: A flat row safe to render as a table. + :rtype: Dict[str, Any] + """ + if not with_history: + return failure + row = {key: value for key, value in failure.items() if key != 'history'} + row['verdict'] = failure.get('history', {}).get('verdict') + return row diff --git a/sp_cli/history.py b/sp_cli/history.py new file mode 100644 index 0000000..4be0008 --- /dev/null +++ b/sp_cli/history.py @@ -0,0 +1,175 @@ +"""Turn a sample's cross-run history into a verdict about *why* it is failing now. + +A classification code (from :mod:`sp_cli.classifier`) says how a test failed. +It cannot say whether the failure is news. That needs the sample's history from +``/samples/{id}/history``, which is what this module reads. + +The distinction the whole module exists for: a test that passed in the previous +run and fails now is a regression someone introduced, and it is the first thing +worth looking at. A test that has never passed is a known gap. Both show up as +plain ``fail`` in a run, and the platform's own UI does not separate them. +""" + +from typing import Any, Dict, List, Optional, Tuple + +#: Passed in the previous run, fails now — the failure this run introduced. +NEW_REGRESSION = 'NEW_REGRESSION' +#: Was already failing before this run, but did pass at some point earlier. +STILL_FAILING = 'STILL_FAILING' +#: No run in the window ever passed — a known gap, not this run's doing. +NEVER_PASSED = 'NEVER_PASSED' +#: Flips between pass and fail often enough that a single flip proves nothing. +FLAKY = 'FLAKY' +#: The sample has no prior run in the window to compare against. +NO_HISTORY = 'NO_HISTORY' +#: History could not be read (no sample id, or the lookup failed). +UNKNOWN = 'UNKNOWN' + +#: Pass/fail flips among prior runs at or above which a failure reads as flaky +#: rather than as a regression. Two flips means the series already alternates, +#: so this run's flip carries no signal. +FLAKY_TRANSITION_THRESHOLD = 2 + + +def _is_pass(entry: Dict[str, Any]) -> bool: + """ + Report whether a history entry records a pass. + + :param entry: One entry from ``/samples/{id}/history``. + :type entry: Dict[str, Any] + :return: True if the entry's status is ``pass``. + :rtype: bool + """ + return entry.get('status') == 'pass' + + +def split_history(entries: List[Dict[str, Any]], run_id: int, + regression_test_id: Optional[int]) -> Tuple[Optional[Dict[str, Any]], + List[Dict[str, Any]]]: + """ + Separate the current run's entry from the runs that came before it. + + ``/samples/{id}/history`` covers every regression test defined for the + sample and includes the run being investigated, so both have to be filtered + out before the remaining entries mean anything. + + :param entries: History entries as returned by the API (newest first). + :type entries: List[Dict[str, Any]] + :param run_id: The run being investigated. + :type run_id: int + :param regression_test_id: Restrict to this regression test, when known. + :type regression_test_id: Optional[int] + :return: The current run's entry (or None) and the prior entries, newest first. + :rtype: Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]]] + """ + if regression_test_id is not None: + entries = [e for e in entries if e.get('regression_test_id') == regression_test_id] + + current = next((e for e in entries if e.get('run_id') == run_id), None) + # Run ids increase over time, so "older" is "lower id". Comparing ids rather + # than tested_at avoids trusting a timestamp that is null for queued runs. + prior = [e for e in entries if isinstance(e.get('run_id'), int) and e['run_id'] < run_id] + return current, prior + + +def _count_transitions(chronological: List[Dict[str, Any]]) -> int: + """ + Count pass/fail flips across a chronologically ordered history. + + :param chronological: Prior entries, oldest first. + :type chronological: List[Dict[str, Any]] + :return: The number of times the outcome changed between runs. + :rtype: int + """ + flips = 0 + for previous, current in zip(chronological, chronological[1:]): + if _is_pass(previous) != _is_pass(current): + flips += 1 + return flips + + +def classify_history(current: Optional[Dict[str, Any]], + prior: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Decide whether a current failure is new, long-standing, never-working, or noise. + + :param current: The current run's history entry, if it was found. + :type current: Optional[Dict[str, Any]] + :param prior: Entries for earlier runs, newest first. + :type prior: List[Dict[str, Any]] + :return: A verdict block with the supporting run ids and signature comparison. + :rtype: Dict[str, Any] + """ + signature = current.get('failure_signature') if current else None + if not prior: + return {'verdict': NO_HISTORY, 'confidence': 'low', + 'reason': 'No earlier run of this test to compare against', + 'last_pass_run': None, 'previous_run': None, 'prior_runs_considered': 0, + 'transitions': 0, 'signature': signature, 'signature_changed': None} + + previous = prior[0] + last_pass = next((e for e in prior if _is_pass(e)), None) + last_fail = next((e for e in prior if not _is_pass(e)), None) + transitions = _count_transitions(list(reversed(prior))) + + signature_changed: Optional[bool] = None + if signature is not None and last_fail is not None: + signature_changed = last_fail.get('failure_signature') != signature + + block = { + 'last_pass_run': last_pass.get('run_id') if last_pass else None, + 'previous_run': previous.get('run_id'), + 'prior_runs_considered': len(prior), + 'transitions': transitions, + 'signature': signature, + 'signature_changed': signature_changed, + } + + if transitions >= FLAKY_TRANSITION_THRESHOLD: + block.update(verdict=FLAKY, confidence='medium', + reason=(f'Flipped between pass and fail {transitions} times in the last ' + f'{len(prior)} runs, so this failure may not be real')) + elif _is_pass(previous): + block.update(verdict=NEW_REGRESSION, confidence='high', + reason=f"Passed in run {previous.get('run_id')}, fails here") + elif last_pass is None: + block.update(verdict=NEVER_PASSED, confidence='high', + reason=f'Has not passed in any of the last {len(prior)} runs') + else: + block.update(verdict=STILL_FAILING, confidence='high', + reason=(f"Already failing in run {previous.get('run_id')}; " + f"last passed in run {last_pass.get('run_id')}")) + return block + + +def unknown_history(reason: str) -> Dict[str, Any]: + """ + Build the verdict block used when history could not be read at all. + + Kept as a real verdict rather than a missing key so every failure row has + the same shape, which is what makes the JSON output safe to iterate over. + + :param reason: Why the lookup did not happen. + :type reason: str + :return: An ``UNKNOWN`` verdict block. + :rtype: Dict[str, Any] + """ + return {'verdict': UNKNOWN, 'confidence': 'low', 'reason': reason, + 'last_pass_run': None, 'previous_run': None, 'prior_runs_considered': 0, + 'transitions': 0, 'signature': None, 'signature_changed': None} + + +def group_by_verdict(failures: List[Dict[str, Any]]) -> Dict[str, int]: + """ + Count classified failures by their history verdict. + + :param failures: Failure rows that each carry a ``history`` block. + :type failures: List[Dict[str, Any]] + :return: A mapping of verdict → count, highest first. + :rtype: Dict[str, int] + """ + counts: Dict[str, int] = {} + for failure in failures: + verdict = failure.get('history', {}).get('verdict', UNKNOWN) + counts[verdict] = counts.get(verdict, 0) + 1 + return dict(sorted(counts.items(), key=lambda item: item[1], reverse=True)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6e898ef..fa0aedb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -195,6 +195,166 @@ def test_investigate_combines_run_summary_and_failures(self, mock_get, mock_pagi self.assertEqual(len(report['failures']), 3) +class InvestigateHistoryTests(unittest.TestCase): + """`investigate --with-history` separates new regressions from known gaps.""" + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @staticmethod + def _history_for(regression_test_id): + """ + Build a plausible history for one regression test. + + :param regression_test_id: The regression test the entries belong to. + :type regression_test_id: int + :return: History entries, newest first. + :rtype: list + """ + return [ + {'run_id': 9299, 'regression_test_id': regression_test_id, 'status': 'fail', + 'failure_signature': 'exit_code_mismatch:rc:10'}, + {'run_id': 9298, 'regression_test_id': regression_test_id, 'status': 'pass', + 'failure_signature': None}, + ] + + def _invoke(self, mock_get, mock_paginated, args, histories): + """ + Run `investigate` with the run/summary/samples calls and histories stubbed. + + :param mock_get: The patched ``ApiClient.get``. + :param mock_paginated: The patched ``ApiClient.get_paginated``. + :param args: CLI arguments. + :type args: list + :param histories: Per-sample history responses, keyed by sample id. + :type histories: dict + :return: The Click result. + """ + mock_get.side_effect = [ + {'run_id': 9299, 'pr_number': 2264, 'platform': 'windows', 'status': 'fail'}, + {'run_id': 9299, 'total_samples': 2, 'pass_count': 0, 'fail_count': 2}, + ] + + def paginated(path, params=None, max_items=1000): + if path.endswith('/samples'): + return SAMPLES_WITH_IDS + sample_id = int(path.split('/samples/')[1].split('/')[0]) + return histories.get(sample_id, []) + + mock_paginated.side_effect = paginated + return self.runner.invoke(cli, args) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_history_labels_regression_versus_never_passing(self, mock_get, mock_paginated): + """A sample that passed last run is a regression; one that never passed is not.""" + result = self._invoke(mock_get, mock_paginated, + ['investigate', '9299', '--with-history'], + {42: self._history_for(18), + 43: [{'run_id': 9298, 'regression_test_id': 137, + 'status': 'fail', 'failure_signature': 'missing_output'}]}) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.output) + verdicts = {f['regression_test_id']: f['history']['verdict'] for f in report['failures']} + self.assertEqual(verdicts, {18: 'NEW_REGRESSION', 137: 'NEVER_PASSED'}) + self.assertEqual(report['by_verdict'], {'NEW_REGRESSION': 1, 'NEVER_PASSED': 1}) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_history_is_filtered_to_the_run_platform(self, mock_get, mock_paginated): + """A Windows failure must not be judged against Linux history.""" + self._invoke(mock_get, mock_paginated, ['investigate', '9299', '--with-history'], + {42: self._history_for(18), 43: self._history_for(137)}) + + history_calls = [c for c in mock_paginated.call_args_list + if '/history' in c.args[0]] + self.assertEqual(len(history_calls), 2) + for call in history_calls: + self.assertEqual(call.kwargs['params']['platform'], 'windows') + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_history_depth_implies_the_flag_and_sets_the_page_size(self, mock_get, mock_paginated): + """--history-depth alone turns history on, and asks for depth + the current run.""" + result = self._invoke(mock_get, mock_paginated, + ['investigate', '9299', '--history-depth', '5'], + {42: self._history_for(18), 43: self._history_for(137)}) + + self.assertEqual(result.exit_code, 0) + self.assertIn('by_verdict', json.loads(result.output)) + history_call = next(c for c in mock_paginated.call_args_list if '/history' in c.args[0]) + self.assertEqual(history_call.kwargs['params']['limit'], 6) + self.assertEqual(history_call.kwargs['max_items'], 6) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_without_the_flag_no_history_is_fetched(self, mock_get, mock_paginated): + """The default stays one call per run — history is opt-in.""" + result = self._invoke(mock_get, mock_paginated, ['investigate', '9299'], {}) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.output) + self.assertNotIn('by_verdict', report) + self.assertNotIn('history', report['failures'][0]) + self.assertFalse([c for c in mock_paginated.call_args_list if '/history' in c.args[0]]) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_one_lookup_per_sample_even_when_tests_share_it(self, mock_get, mock_paginated): + """Two regression tests on one sample must not cost two history calls.""" + mock_get.side_effect = [ + {'run_id': 9299, 'platform': 'windows', 'status': 'fail'}, + {'run_id': 9299, 'total_samples': 2, 'pass_count': 0, 'fail_count': 2}, + ] + shared = [dict(s, sample_id=42) for s in SAMPLES_WITH_IDS] + + def paginated(path, params=None, max_items=1000): + if path.endswith('/samples'): + return shared + return self._history_for(18) + self._history_for(137) + + mock_paginated.side_effect = paginated + result = self.runner.invoke(cli, ['investigate', '9299', '--with-history']) + + self.assertEqual(result.exit_code, 0) + history_calls = [c for c in mock_paginated.call_args_list if '/history' in c.args[0]] + self.assertEqual(len(history_calls), 1) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_missing_sample_id_yields_unknown_not_a_crash(self, mock_get, mock_paginated): + """A result with no sample id still gets a verdict block.""" + mock_get.side_effect = [ + {'run_id': 9299, 'platform': 'windows', 'status': 'fail'}, + {'run_id': 9299, 'total_samples': 1, 'pass_count': 0, 'fail_count': 1}, + ] + mock_paginated.return_value = [ + {'regression_test_id': 18, 'sample_id': None, 'sample_name': 'orphan', + 'categories': [], 'status': 'fail', 'exit_code': 1, 'expected_rc': 0, 'outputs': []}, + ] + result = self.runner.invoke(cli, ['investigate', '9299', '--with-history']) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.output) + self.assertEqual(report['failures'][0]['history']['verdict'], 'UNKNOWN') + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_table_output_lifts_the_verdict_into_a_column(self, mock_get, mock_paginated): + """The nested block is right for JSON and wrong for a table cell.""" + result = self._invoke(mock_get, mock_paginated, + ['-o', 'table', 'investigate', '9299', '--with-history'], + {42: self._history_for(18), 43: self._history_for(137)}) + + self.assertEqual(result.exit_code, 0) + self.assertIn('verdict', result.output) + self.assertIn('NEW_REGRESSION', result.output) + self.assertIn('by history:', result.output) + self.assertNotIn("{'verdict'", result.output) + + class ApiContractTests(unittest.TestCase): """Pin the CLI surface to what the merged ``mod_api`` blueprint actually accepts. diff --git a/tests/test_history.py b/tests/test_history.py new file mode 100644 index 0000000..374608d --- /dev/null +++ b/tests/test_history.py @@ -0,0 +1,165 @@ +"""Tests for the cross-run history verdicts behind ``sp investigate --with-history``.""" + +import unittest + +from sp_cli.history import (FLAKY, NEVER_PASSED, NEW_REGRESSION, NO_HISTORY, + STILL_FAILING, UNKNOWN, classify_history, + group_by_verdict, split_history, unknown_history) + + +def entry(run_id, status, regression_test_id=137, signature=None): + """ + Build one history entry shaped like ``/samples/{id}/history`` returns. + + :param run_id: The run the entry belongs to. + :type run_id: int + :param status: Derived per-sample status for that run. + :type status: str + :param regression_test_id: Which regression test the entry is for. + :type regression_test_id: int + :param signature: Optional failure signature. + :type signature: Optional[str] + :return: A history entry. + :rtype: dict + """ + return {'run_id': run_id, 'regression_test_id': regression_test_id, 'status': status, + 'platform': 'windows', 'branch': 'master', 'commit_sha': 'abc1234', + 'tested_at': '2026-07-30T10:00:00Z', 'failure_signature': signature} + + +class SplitHistoryTests(unittest.TestCase): + """The endpoint returns more than the run and test being investigated.""" + + def test_drops_other_regression_tests_for_the_same_sample(self): + """One sample can back several regression tests; only the failing one counts.""" + entries = [entry(9299, 'fail'), entry(9298, 'pass'), + entry(9298, 'fail', regression_test_id=999)] + current, prior = split_history(entries, 9299, 137) + + self.assertEqual(current['run_id'], 9299) + self.assertEqual([e['run_id'] for e in prior], [9298]) + self.assertTrue(all(e['regression_test_id'] == 137 for e in prior)) + + def test_excludes_the_run_being_investigated_from_prior(self): + """The current run appears in its own history and must not count as prior.""" + entries = [entry(9299, 'fail'), entry(9298, 'pass'), entry(9290, 'pass')] + current, prior = split_history(entries, 9299, 137) + + self.assertEqual(current['run_id'], 9299) + self.assertEqual([e['run_id'] for e in prior], [9298, 9290]) + + def test_ignores_runs_newer_than_the_one_investigated(self): + """Investigating an older run must not be judged against runs that came after it.""" + entries = [entry(9300, 'pass'), entry(9299, 'fail'), entry(9298, 'pass')] + _, prior = split_history(entries, 9299, 137) + + self.assertEqual([e['run_id'] for e in prior], [9298]) + + def test_keeps_every_entry_when_no_regression_test_is_known(self): + """Without a regression test id there is nothing to narrow by.""" + entries = [entry(9299, 'fail'), entry(9298, 'pass', regression_test_id=999)] + _, prior = split_history(entries, 9299, None) + + self.assertEqual(len(prior), 1) + + +class ClassifyHistoryTests(unittest.TestCase): + """The verdict is what separates 'someone broke this' from 'this never worked'.""" + + def test_no_prior_runs(self): + """A sample with no earlier run cannot be called a regression.""" + block = classify_history(entry(9299, 'fail'), []) + + self.assertEqual(block['verdict'], NO_HISTORY) + self.assertEqual(block['prior_runs_considered'], 0) + self.assertIsNone(block['signature_changed']) + + def test_new_regression_when_the_previous_run_passed(self): + """Passed last run, fails now — the case worth looking at first.""" + block = classify_history(entry(9299, 'fail'), + [entry(9298, 'pass'), entry(9290, 'pass')]) + + self.assertEqual(block['verdict'], NEW_REGRESSION) + self.assertEqual(block['confidence'], 'high') + self.assertEqual(block['previous_run'], 9298) + self.assertEqual(block['last_pass_run'], 9298) + + def test_never_passed_when_no_prior_run_passed(self): + """A known gap, not something this run introduced.""" + block = classify_history(entry(9299, 'fail'), + [entry(9298, 'fail'), entry(9290, 'missing_output')]) + + self.assertEqual(block['verdict'], NEVER_PASSED) + self.assertIsNone(block['last_pass_run']) + + def test_still_failing_when_it_broke_before_this_run(self): + """Already broken on arrival, but it did work at some point.""" + block = classify_history(entry(9299, 'fail'), + [entry(9298, 'fail'), entry(9290, 'pass')]) + + self.assertEqual(block['verdict'], STILL_FAILING) + self.assertEqual(block['last_pass_run'], 9290) + self.assertEqual(block['previous_run'], 9298) + + def test_flaky_beats_regression_when_the_series_alternates(self): + """A pass/fail flip proves nothing in a series that already flips.""" + block = classify_history(entry(9299, 'fail'), + [entry(9298, 'pass'), entry(9297, 'fail'), + entry(9296, 'pass')]) + + self.assertEqual(block['verdict'], FLAKY) + self.assertEqual(block['transitions'], 2) + + def test_single_flip_is_still_a_regression(self): + """One transition is the regression case, not the flaky one.""" + block = classify_history(entry(9299, 'fail'), + [entry(9298, 'pass'), entry(9297, 'pass'), + entry(9296, 'fail')]) + + self.assertEqual(block['transitions'], 1) + self.assertEqual(block['verdict'], NEW_REGRESSION) + + def test_signature_change_is_reported_against_the_last_failure(self): + """Same code, different signature means a different underlying failure.""" + changed = classify_history( + entry(9299, 'fail', signature='exit_code_mismatch:rc:10'), + [entry(9298, 'fail', signature='missing_output')]) + unchanged = classify_history( + entry(9299, 'fail', signature='missing_output'), + [entry(9298, 'fail', signature='missing_output')]) + + self.assertTrue(changed['signature_changed']) + self.assertFalse(unchanged['signature_changed']) + + def test_signature_unknown_when_the_current_entry_is_missing(self): + """History may not include the current run; the verdict still works.""" + block = classify_history(None, [entry(9298, 'pass')]) + + self.assertEqual(block['verdict'], NEW_REGRESSION) + self.assertIsNone(block['signature']) + self.assertIsNone(block['signature_changed']) + + +class VerdictGroupingTests(unittest.TestCase): + """The tally is what makes a 40-failure run readable.""" + + def test_group_by_verdict_counts_highest_first(self): + """Counts sort descending so the dominant verdict leads.""" + failures = [ + {'history': {'verdict': NEVER_PASSED}}, + {'history': {'verdict': NEVER_PASSED}}, + {'history': {'verdict': NEW_REGRESSION}}, + ] + + self.assertEqual(group_by_verdict(failures), + {NEVER_PASSED: 2, NEW_REGRESSION: 1}) + + def test_rows_without_history_count_as_unknown(self): + """Every row is counted, so the tally always sums to the failure count.""" + self.assertEqual(group_by_verdict([{}, {'history': {}}]), {UNKNOWN: 2}) + + def test_unknown_history_has_the_same_shape_as_a_real_verdict(self): + """Uniform keys are what make the JSON safe for an agent to iterate.""" + real = classify_history(entry(9299, 'fail'), [entry(9298, 'pass')]) + + self.assertEqual(set(unknown_history('no id')), set(real)) From aebdce43b6529ba200aa7aeabba450dc6a536bae Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 5 Aug 2026 23:26:36 +0530 Subject: [PATCH 04/10] feat: run errors, logs, artifacts, error-summary, infra-errors sample-platform#1135 (errors, logs) and #1141 (test artifacts) are merged, so the three commands that shipped as PENDING stubs are now real, and the endpoints they left uncovered get commands of their own. sp run error-summary grouped counts -- the cheapest first look sp run errors per-test errors, --type/--severity/--sample sp run infra-errors VM / checkout / build / worker / storage sp run logs build log, --level/--source/--contains sp run artifacts binary, coredump, outputs, build log Three contract details that shaped this: - /runs/{id}/logs is the only cursor-paginated endpoint; sending it an offset is a 400 ("Cannot mix cursor and offset pagination"). Hence client.get_cursor_paginated alongside get_paginated, and --cursor/--all rather than --offset. The table footer reported next_offset and silently dropped next_cursor, so a partial log looked complete. - A missing log answers 404 with code log_not_found, distinct from a missing run's not_found, and points at the artifacts endpoint. Both map to exit code 4, so callers must branch on error.code, not the status. - The old --type help advertised 'test_failure', which the API cannot emit. Test errors are derived per request and are only ever exit_code_mismatch, missing_output, or diff_mismatch. /runs/{id}/samples/{sid}/logs deliberately gets no command: it is a permanent 404 by design, because the CI worker does not store per-sample logs. --- sp_cli/client.py | 35 ++++++++++ sp_cli/commands/run.py | 154 +++++++++++++++++++++++++++++++++++++---- sp_cli/output.py | 2 + tests/test_cli.py | 142 +++++++++++++++++++++++++++++++++++++ 4 files changed, 320 insertions(+), 13 deletions(-) diff --git a/sp_cli/client.py b/sp_cli/client.py index 2719dee..dc9bb5f 100644 --- a/sp_cli/client.py +++ b/sp_cli/client.py @@ -166,3 +166,38 @@ def get_paginated(self, path: str, params: Optional[Dict[str, Any]] = None, break offset = next_offset return items + + def get_cursor_paginated(self, path: str, params: Optional[Dict[str, Any]] = None, + max_items: int = 5000) -> List[Any]: + """ + Follow cursor pagination and return the combined ``data`` list. + + Distinct from :meth:`get_paginated` because the API refuses to mix the + two schemes: sending ``offset`` to a cursor-paginated endpoint is a 400. + The cursor is opaque here -- it is echoed back exactly as received. + + :param path: API path below ``/api/v1``. + :type path: str + :param params: Optional query-string parameters (``cursor`` is managed). + :type params: Optional[Dict[str, Any]] + :param max_items: Safety cap on total items collected. + :type max_items: int + :return: All items across pages. + :rtype: List[Any] + """ + base = dict(params or {}) + cursor = base.pop('cursor', None) + items: List[Any] = [] + while True: + # A fresh mapping per page: reusing one would alias into every call. + page_params = dict(base) + if cursor is not None: + page_params['cursor'] = cursor + payload = self.get(path, params=page_params) + data = payload.get('data', []) if isinstance(payload, dict) else [] + items.extend(data) + pagination = payload.get('pagination', {}) if isinstance(payload, dict) else {} + cursor = pagination.get('next_cursor') + if not data or cursor is None or len(items) >= max_items: + break + return items diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index 6f5ec54..cf89480 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -5,8 +5,11 @@ import click from sp_cli.client import ApiError -from sp_cli.constants import (CANCEL_REASON_MIN_LENGTH, PLATFORMS, - RUN_STATUSES, SAMPLE_STATUSES) +from sp_cli.constants import (ARTIFACT_TYPES, CANCEL_REASON_MIN_LENGTH, + ERROR_GROUP_BY, ERROR_SEVERITIES, ERROR_TYPES, + INFRA_ERROR_TYPES, LOG_CONTAINS_MAX_LENGTH, + LOG_LEVELS, LOG_MAX_LIMIT, LOG_SOURCES, + PLATFORMS, RUN_STATUSES, SAMPLE_STATUSES) from sp_cli.output import render, render_error from sp_cli.runner import clean_params, fetch_and_render from sp_cli.triage import classify_sample, is_failure @@ -277,28 +280,153 @@ def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: O @run.command('artifacts') @click.argument('run_id', type=int) +@click.option('--type', 'artifact_type', type=click.Choice(ARTIFACT_TYPES), default=None, + help='Filter by artifact type.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context -def run_artifacts(ctx: click.Context, run_id: int) -> None: - """List downloadable artifacts for a run (signed URLs).""" - fetch_and_render(ctx, f'/runs/{run_id}/artifacts') +def run_artifacts(ctx: click.Context, run_id: int, artifact_type: Optional[str], + limit: Optional[int], offset: Optional[int]) -> None: + """List downloadable artifacts for a run. + + Covers the build binary, any coredump, combined stdout, the build log, and + every expected/actual output file. `storage_status` says whether the blob is + actually retrievable; `download_url` is null for the build log, which is read + through `sp run logs` instead of downloaded. + """ + params = clean_params({'type': artifact_type, 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, f'/runs/{run_id}/artifacts', params) @run.command('logs') @click.argument('run_id', type=int) +@click.option('--level', type=click.Choice(LOG_LEVELS), default=None, + help='Keep lines at this level. Levels are scanned out of the raw text.') +@click.option('--source', type=click.Choice(LOG_SOURCES), default=None, + help='Keep lines from this component.') +@click.option('--contains', default=None, + help=f'Case-insensitive substring filter (max {LOG_CONTAINS_MAX_LENGTH} chars).') +@click.option('--limit', type=int, default=None, + help=f'Lines per page (max {LOG_MAX_LIMIT}, default 100).') +@click.option('--cursor', default=None, help='Resume from a previous response\'s next_cursor.') +@click.option('--all', 'fetch_all', is_flag=True, default=False, + help='Follow next_cursor and return the whole log at once.') @click.pass_context -def run_logs(ctx: click.Context, run_id: int) -> None: - """Show raw logs for a run (requires contributor or admin privileges).""" - fetch_and_render(ctx, f'/runs/{run_id}/logs') +def run_logs(ctx: click.Context, run_id: int, level: Optional[str], source: Optional[str], + contains: Optional[str], limit: Optional[int], cursor: Optional[str], + fetch_all: bool) -> None: + """Read a run's build log (requires the system:read scope). + + This endpoint is cursor-paginated, not offset-paginated: page forward with + --cursor using the next_cursor from the previous response, or pass --all to + follow it to the end. Sending an offset is a 400. + + A 404 with code `log_not_found` means the log is no longer on the VM's disk + rather than that the run is missing -- fetch it from `sp run artifacts + --type build_log` in that case. + + The level and source of each line are recovered by scanning the raw text, so + they are best-effort: an unrecognized line reports level=info, source=web. + """ + if contains is not None and len(contains) > LOG_CONTAINS_MAX_LENGTH: + raise click.BadParameter( + f'must be {LOG_CONTAINS_MAX_LENGTH} characters or less', param_hint='--contains') + if fetch_all and cursor is not None: + raise click.BadParameter('--all starts from the beginning; drop --cursor', + param_hint='--cursor') + + params = clean_params({'level': level, 'source': source, 'contains': contains, + 'limit': limit, 'cursor': cursor}) + if not fetch_all: + fetch_and_render(ctx, f'/runs/{run_id}/logs', params) + return + + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + lines = client.get_cursor_paginated(f'/runs/{run_id}/logs', params) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + render({'data': lines, 'summary': {'lines': len(lines)}}, output) @run.command('errors') @click.argument('run_id', type=int) -@click.option('--type', 'error_type', default=None, - help='test_failure|exit_code_mismatch|missing_output|diff_mismatch') +@click.option('--type', 'error_type', type=click.Choice(ERROR_TYPES), default=None, + help='Filter by error type.') +@click.option('--severity', type=click.Choice(ERROR_SEVERITIES), default=None, + help='Filter by severity. Test errors are only error or warning.') +@click.option('--sample', 'sample_id', type=int, default=None, + help='Restrict to one media sample id.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def run_errors(ctx: click.Context, run_id: int, error_type: Optional[str], + severity: Optional[str], sample_id: Optional[int], + limit: Optional[int], offset: Optional[int]) -> None: + """Show structured test errors for a run. + + These are derived from the result rows at request time rather than stored, + so they line up with `sp run failures` but carry per-output detail. For + failures of the infrastructure itself, see `sp run infra-errors`. + """ + params = clean_params({'type': error_type, 'severity': severity, 'sample_id': sample_id, + 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, f'/runs/{run_id}/errors', params) + + +@run.command('error-summary') +@click.argument('run_id', type=int) +@click.option('--group-by', 'group_by', type=click.Choice(ERROR_GROUP_BY), default=None, + help='Bucket key. The API defaults to type.') +@click.option('--severity', type=click.Choice(ERROR_SEVERITIES), default=None, + help='Keep only buckets at this severity.') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') @click.pass_context -def run_errors(ctx: click.Context, run_id: int, error_type: Optional[str]) -> None: - """Show structured test errors for a run.""" - fetch_and_render(ctx, f'/runs/{run_id}/errors', clean_params({'type': error_type})) +def run_error_summary(ctx: click.Context, run_id: int, group_by: Optional[str], + severity: Optional[str], limit: Optional[int], + offset: Optional[int]) -> None: + """Show grouped error counts for a run. + + The cheapest first look at a broken run: one row per bucket with a count, + so an agent can decide what to drill into before pulling every error. + Each bucket's severity is the highest of the errors within it. + """ + params = clean_params({'group_by': group_by, 'severity': severity, + 'limit': limit, 'offset': offset}) + fetch_and_render(ctx, f'/runs/{run_id}/error-summary', params) + + +@run.command('infra-errors') +@click.argument('run_id', type=int) +@click.option('--type', 'error_type', type=click.Choice(INFRA_ERROR_TYPES), default=None, + help='Filter by infrastructure failure stage.') +@click.option('--severity', type=click.Choice(ERROR_SEVERITIES), default=None, + help='Filter by severity. These are always reported as critical.') +@click.option('--include-stack', is_flag=True, default=False, + help='Include stack traces (admin or contributor only; 403 otherwise).') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def run_infra_errors(ctx: click.Context, run_id: int, error_type: Optional[str], + severity: Optional[str], include_stack: bool, + limit: Optional[int], offset: Optional[int]) -> None: + """Show infrastructure errors for a run (requires the system:read scope). + + These are VM, checkout, build, worker, and storage failures, classified from + TestProgress messages by keyword -- so the type is a best-effort guess, not a + recorded field. A run that fails here usually has no test errors at all. + + Stack traces are withheld by default because they carry internal paths; + --include-stack needs the admin or contributor role and 403s without it. + """ + params = clean_params({'type': error_type, 'severity': severity, + 'limit': limit, 'offset': offset}) + if include_stack: + params['include_stack'] = 'true' + fetch_and_render(ctx, f'/runs/{run_id}/infrastructure-errors', params) def _resolve_diff_targets(client: Any, run_id: int, sample_id: int, diff --git a/sp_cli/output.py b/sp_cli/output.py index cd5e0d2..bf61a32 100644 --- a/sp_cli/output.py +++ b/sp_cli/output.py @@ -74,6 +74,8 @@ def _footer(payload: Dict[str, Any]) -> str: parts.append(f"{pagination['total']} total") if pagination.get('next_offset') is not None: parts.append(f"more at offset {pagination['next_offset']}") + if pagination.get('next_cursor') is not None: + parts.append(f"more at cursor {pagination['next_cursor']}") return ' · '.join(parts) return '' diff --git a/tests/test_cli.py b/tests/test_cli.py index fa0aedb..8ac5e51 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -540,3 +540,145 @@ def test_queue_rejects_non_queue_status_and_paginates(self, mock_get): result = self.runner.invoke(cli, ['queue', '--status', 'queued', '--limit', '10']) self.assertEqual(result.exit_code, 0) mock_get.assert_called_once_with('/system/queue', params={'status': 'queued', 'limit': 10}) + + +class ErrorsLogsArtifactsTests(unittest.TestCase): + """Cover the endpoints unblocked by sample-platform#1135 and #1141. + + These commands shipped as stubs while their PRs were open; the assertions + here pin them to the contracts that actually landed. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_errors_forwards_every_declared_filter(self, mock_get): + """`run errors` supports the type, severity, and sample_id filters the route applies.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, [ + 'run', 'errors', '9299', '--type', 'missing_output', + '--severity', 'error', '--sample', '42', '--limit', '10']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/errors', params={ + 'type': 'missing_output', 'severity': 'error', 'sample_id': 42, 'limit': 10}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_errors_rejects_types_the_service_cannot_emit(self, mock_get): + """derive_errors_for_run only produces three types; 'test_failure' is not one.""" + for bad_type in ('test_failure', 'segfault', 'timeout'): + result = self.runner.invoke(cli, ['run', 'errors', '9299', '--type', bad_type]) + self.assertNotEqual(result.exit_code, 0, f'{bad_type} should be rejected') + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_error_summary_group_by_is_a_closed_set(self, mock_get): + """group_by outside the API's four keys is a 400, so reject it locally.""" + self.assertNotEqual( + self.runner.invoke(cli, ['run', 'error-summary', '9299', '--group-by', 'run']).exit_code, 0) + mock_get.assert_not_called() + + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['run', 'error-summary', '9299', '--group-by', 'regression_id']) + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/error-summary', + params={'group_by': 'regression_id'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_infra_errors_target_the_hyphenated_path_and_gate_stacks(self, mock_get): + """The route is /infrastructure-errors, and include_stack is opt-in as a string.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['run', 'infra-errors', '9299', '--type', 'vm_provisioning']) + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/infrastructure-errors', + params={'type': 'vm_provisioning'}) + + mock_get.reset_mock() + result = self.runner.invoke(cli, ['run', 'infra-errors', '9299', '--include-stack']) + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/infrastructure-errors', + params={'include_stack': 'true'}) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_infra_error_type_is_restricted_to_the_classifier_buckets(self, mock_get): + """_classify_infra_error only ever returns six values.""" + result = self.runner.invoke(cli, ['run', 'infra-errors', '9299', '--type', 'network']) + self.assertNotEqual(result.exit_code, 0) + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_artifacts_filters_by_type(self, mock_get): + """`run artifacts --type` mirrors the artifact kinds list_artifacts builds.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['run', 'artifacts', '9299', '--type', 'coredump']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/artifacts', params={'type': 'coredump'}) + + self.assertNotEqual( + self.runner.invoke(cli, ['run', 'artifacts', '9299', '--type', 'stderr']).exit_code, 0) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_logs_use_cursor_pagination_never_offset(self, mock_get): + """The logs route 400s if offset is present, so the CLI exposes --cursor only.""" + mock_get.return_value = {'data': [], 'pagination': {'limit': 100, 'next_cursor': None}} + result = self.runner.invoke(cli, [ + 'run', 'logs', '9299', '--level', 'error', '--source', 'worker', + '--contains', 'segfault', '--limit', '50', '--cursor', '400']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/runs/9299/logs', params={ + 'level': 'error', 'source': 'worker', 'contains': 'segfault', + 'limit': 50, 'cursor': '400'}) + + self.assertNotEqual( + self.runner.invoke(cli, ['run', 'logs', '9299', '--offset', '10']).exit_code, 0) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_logs_rejects_an_over_long_contains_before_the_request(self, mock_get): + """The API caps contains at 100 characters; fail locally instead of round-tripping.""" + result = self.runner.invoke(cli, ['run', 'logs', '9299', '--contains', 'x' * 101]) + + self.assertNotEqual(result.exit_code, 0) + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_logs_all_follows_the_cursor_to_the_end(self, mock_get): + """--all pages through next_cursor and collapses the result into one list.""" + mock_get.side_effect = [ + {'data': [{'message': 'first'}], 'pagination': {'next_cursor': '1'}}, + {'data': [{'message': 'second'}], 'pagination': {'next_cursor': None}}, + ] + result = self.runner.invoke(cli, ['run', 'logs', '9299', '--all']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(mock_get.call_args_list, [ + mock.call('/runs/9299/logs', params={}), + mock.call('/runs/9299/logs', params={'cursor': '1'}), + ]) + payload = json.loads(result.output) + self.assertEqual([line['message'] for line in payload['data']], ['first', 'second']) + self.assertEqual(payload['summary']['lines'], 2) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_logs_all_and_cursor_are_mutually_exclusive(self, mock_get): + """--all restarts from the top, so combining it with --cursor is a usage error.""" + result = self.runner.invoke(cli, ['run', 'logs', '9299', '--all', '--cursor', '40']) + + self.assertNotEqual(result.exit_code, 0) + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_missing_log_file_is_distinguishable_from_a_missing_run(self, mock_get): + """A cold-storage log 404s as log_not_found, which the envelope must preserve.""" + mock_get.side_effect = ApiError( + 'log_not_found', 'Log file for run 9299 is not available locally.', 404, + {'run_id': 9299, 'action_required': 'Use GET /runs/9299/artifacts (type=build_log)'}) + result = self.runner.invoke(cli, ['run', 'logs', '9299']) + + self.assertEqual(result.exit_code, 4) + envelope = json.loads(result.stderr) + self.assertEqual(envelope['error']['code'], 'log_not_found') + self.assertIn('artifacts', envelope['error']['details']['action_required']) From 02a578c2d3eaf6541567f7a30577dcc109b905b1 Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 5 Aug 2026 23:31:24 +0530 Subject: [PATCH 05/10] feat: run create, regression/category CRUD, user and admin commands Covers every remaining merged endpoint, so the CLI is no longer read-mostly: sp run create POST /runs -- queue a run sp regression show/create/edit/rm sp category ls/create/edit/rm sp sample details upload record, extra files, media info sp auth whoami/users/set-role sp admin maintenance/pause/resume sp admin blocked-users ls/add/rm sp admin forbidden-extensions ls/add/rm 409 conflict now maps to its own exit code (8). A refused delete is not a validation failure -- the body was fine, the world disagreed -- and callers need to tell "this test still has 23 results" apart from "your request was malformed". Both delete paths return it, and both suggest the alternative: retire with `edit --inactive` rather than deleting. Request-shape details that are easy to get wrong, all pinned by tests: - Regression-test categories are given by name, not id, must already exist, and on PATCH the list is replaced rather than merged. - A new regression test is created inactive unless --active, matching the API's default: a maintainer should see what it produces on a verification run before it joins the suite. So `active` is sent only when asked for, otherwise "off" is indistinguishable from "unset". - POST /runs needs a full 40-char commit_sha and owner/repo -- both rejected locally -- and schedules a test of an existing CI artifact rather than triggering a compile. - Forbidden extensions are stored lower-cased without a leading dot, so .MKV and mkv normalize to the same request. - Blocked users key on the numeric GitHub id, not the login: logins can be changed and reused, which would silently unblock somebody. PATCH bodies are sparse -- only the flags you pass are sent -- and an empty edit is a usage error rather than a pointless round trip. --- sp_cli/client.py | 44 +++++- sp_cli/commands/admin.py | 139 ++++++++++++++++++ sp_cli/commands/auth.py | 46 +++++- sp_cli/commands/category.py | 83 +++++++++++ sp_cli/commands/regression.py | 124 +++++++++++++++- sp_cli/commands/run.py | 50 ++++++- sp_cli/commands/sample.py | 13 ++ sp_cli/main.py | 4 + sp_cli/runner.py | 26 ++++ tests/test_cli.py | 258 ++++++++++++++++++++++++++++++++++ 10 files changed, 773 insertions(+), 14 deletions(-) create mode 100644 sp_cli/commands/admin.py create mode 100644 sp_cli/commands/category.py diff --git a/sp_cli/client.py b/sp_cli/client.py index dc9bb5f..3f8c6a7 100644 --- a/sp_cli/client.py +++ b/sp_cli/client.py @@ -33,7 +33,8 @@ def exit_code(self) -> int: """ Map the error to a process exit code so callers can branch on it. - :return: 3 connection · 4 not-found · 5 validation · 6 auth · 7 rate-limited · 1 other. + :return: 3 connection · 4 not-found · 5 validation · 6 auth · 7 rate-limited · + 8 conflict · 1 other. :rtype: int """ if self.code == 'connection_error': @@ -46,6 +47,10 @@ def exit_code(self) -> int: return 6 if self.status == 429: return 7 + # A refused delete (test/category still referenced) or a duplicate name. + # Distinct from validation: the body was fine, the world disagreed. + if self.status == 409: + return 8 return 1 @@ -137,6 +142,43 @@ def get(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: """ return self.request('GET', path, params=params) + def post(self, path: str, json_body: Optional[Dict[str, Any]] = None) -> Any: + """ + Perform a POST and return the decoded body. + + :param path: API path below ``/api/v1``. + :type path: str + :param json_body: Optional JSON request body. + :type json_body: Optional[Dict[str, Any]] + :return: The decoded JSON body. + :rtype: Any + """ + return self.request('POST', path, json_body=json_body) + + def patch(self, path: str, json_body: Optional[Dict[str, Any]] = None) -> Any: + """ + Perform a PATCH and return the decoded body. + + :param path: API path below ``/api/v1``. + :type path: str + :param json_body: Optional JSON request body. + :type json_body: Optional[Dict[str, Any]] + :return: The decoded JSON body. + :rtype: Any + """ + return self.request('PATCH', path, json_body=json_body) + + def delete(self, path: str) -> Any: + """ + Perform a DELETE and return the decoded body. + + :param path: API path below ``/api/v1``. + :type path: str + :return: The decoded JSON body (or ``None`` for ``204``). + :rtype: Any + """ + return self.request('DELETE', path) + def get_paginated(self, path: str, params: Optional[Dict[str, Any]] = None, max_items: int = 1000) -> List[Any]: """ diff --git a/sp_cli/commands/admin.py b/sp_cli/commands/admin.py new file mode 100644 index 0000000..3f76e83 --- /dev/null +++ b/sp_cli/commands/admin.py @@ -0,0 +1,139 @@ +"""``sp admin`` — platform configuration: maintenance, blocked users, forbidden extensions. + +Every command here needs the admin role, and the write ones additionally need +the ``system:write`` scope. A token without them gets a 403 (exit code 6). +""" + +from typing import Optional + +import click + +from sp_cli.constants import (BLOCKED_USER_COMMENT_MAX_LENGTH, + EXTENSION_MAX_LENGTH, PLATFORMS) +from sp_cli.runner import clean_params, fetch_and_render, send_and_render + + +@click.group() +def admin() -> None: + """Inspect and change platform configuration (admin only).""" + + +@admin.command('maintenance') +@click.pass_context +def admin_maintenance(ctx: click.Context) -> None: + """Show whether CI is paused, one entry per platform. + + Answers under a ``platforms`` key rather than the usual ``data`` collection + envelope: the list comes from the platform enum, not a growable table, so + there is nothing to paginate. + """ + fetch_and_render(ctx, '/system/maintenance') + + +@admin.command('pause') +@click.argument('platform', type=click.Choice(PLATFORMS)) +@click.pass_context +def admin_pause(ctx: click.Context, platform: str) -> None: + """Stop handing new runs to this platform's VMs. + + Runs still queue while a platform is paused -- they are simply not + dispatched until it is resumed. + """ + send_and_render(ctx, 'PATCH', f'/system/maintenance/{platform}', {'disabled': True}) + + +@admin.command('resume') +@click.argument('platform', type=click.Choice(PLATFORMS)) +@click.pass_context +def admin_resume(ctx: click.Context, platform: str) -> None: + """Resume dispatching runs to this platform's VMs.""" + send_and_render(ctx, 'PATCH', f'/system/maintenance/{platform}', {'disabled': False}) + + +@admin.group('blocked-users') +def blocked_users() -> None: + """List, block, and unblock GitHub accounts.""" + + +@blocked_users.command('ls') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def blocked_users_ls(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: + """List the GitHub accounts blocked from triggering CI runs.""" + fetch_and_render(ctx, '/system/blocked-users', + clean_params({'limit': limit, 'offset': offset})) + + +@blocked_users.command('add') +@click.argument('user_id', type=int) +@click.option('--comment', default=None, + help=f'Why the account is blocked (max {BLOCKED_USER_COMMENT_MAX_LENGTH} chars).') +@click.pass_context +def blocked_users_add(ctx: click.Context, user_id: int, comment: Optional[str]) -> None: + """Block a GitHub account from triggering CI runs. + + USER_ID is the *numeric* GitHub account id, not the login: logins can be + changed and reused, which would silently unblock somebody. Blocking an + already-blocked account is a 409 (exit code 8). + """ + if comment is not None and len(comment) > BLOCKED_USER_COMMENT_MAX_LENGTH: + raise click.BadParameter( + f'must be {BLOCKED_USER_COMMENT_MAX_LENGTH} characters or less', + param_hint='--comment') + send_and_render(ctx, 'POST', '/system/blocked-users', + clean_params({'user_id': user_id, 'comment': comment})) + + +@blocked_users.command('rm') +@click.argument('user_id', type=int) +@click.pass_context +def blocked_users_rm(ctx: click.Context, user_id: int) -> None: + """Unblock a GitHub account (by numeric id).""" + send_and_render(ctx, 'DELETE', f'/system/blocked-users/{user_id}') + + +@admin.group('forbidden-extensions') +def forbidden_extensions() -> None: + """List, forbid, and re-allow upload file extensions.""" + + +@forbidden_extensions.command('ls') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def forbidden_extensions_ls(ctx: click.Context, limit: Optional[int], + offset: Optional[int]) -> None: + """List the file extensions rejected on upload. + + Answers a list of bare strings rather than objects, so table output shows a + single unnamed column. + """ + fetch_and_render(ctx, '/system/forbidden-extensions', + clean_params({'limit': limit, 'offset': offset})) + + +@forbidden_extensions.command('add') +@click.argument('extension') +@click.pass_context +def forbidden_extensions_add(ctx: click.Context, extension: str) -> None: + """Forbid an extension from being uploaded. + + Give it without the leading dot; it is stored lower-cased. Letters and + digits only, so a glob or pattern can never be smuggled in. + """ + normalized = extension.lstrip('.').lower() + if not normalized.isalnum() or not 1 <= len(normalized) <= EXTENSION_MAX_LENGTH: + raise click.BadParameter( + f'must be 1 to {EXTENSION_MAX_LENGTH} alphanumeric characters, without a leading dot', + param_hint='EXTENSION') + send_and_render(ctx, 'POST', '/system/forbidden-extensions', {'extension': normalized}) + + +@forbidden_extensions.command('rm') +@click.argument('extension') +@click.pass_context +def forbidden_extensions_rm(ctx: click.Context, extension: str) -> None: + """Allow an extension to be uploaded again.""" + send_and_render(ctx, 'DELETE', + f'/system/forbidden-extensions/{extension.lstrip(".").lower()}') diff --git a/sp_cli/commands/auth.py b/sp_cli/commands/auth.py index 0b38223..7fab29c 100644 --- a/sp_cli/commands/auth.py +++ b/sp_cli/commands/auth.py @@ -1,18 +1,19 @@ -"""``sp auth`` — obtain, list, and revoke API tokens.""" +"""``sp auth`` — obtain, list, and revoke API tokens, and manage users.""" from typing import Optional, Tuple import click from sp_cli.client import ApiError -from sp_cli.constants import TOKEN_MAX_DAYS, TOKEN_MIN_DAYS, TOKEN_SCOPES +from sp_cli.constants import (TOKEN_MAX_DAYS, TOKEN_MIN_DAYS, TOKEN_SCOPES, + USER_ROLES) from sp_cli.output import render, render_error -from sp_cli.runner import clean_params, fetch_and_render +from sp_cli.runner import clean_params, fetch_and_render, send_and_render @click.group() def auth() -> None: - """Obtain, list, and revoke API tokens.""" + """Obtain, list, and revoke API tokens, and manage users.""" @auth.command('login') @@ -88,3 +89,40 @@ def auth_logout(ctx: click.Context) -> None: render_error(error, output) raise SystemExit(error.exit_code) click.echo('Token revoked.') + + +@auth.command('whoami') +@click.pass_context +def auth_whoami(ctx: click.Context) -> None: + """Show the account and role the current token authenticates as. + + The cheapest way to check that a token is live and carries the scopes a + command needs, without making a change. + """ + fetch_and_render(ctx, '/auth/me') + + +@auth.command('users') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def auth_users(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: + """List platform users, oldest first (admin only). + + Needs the tokens:manage scope as well as the admin role. Password hashes + and GitHub tokens are never included. + """ + fetch_and_render(ctx, '/users', clean_params({'limit': limit, 'offset': offset})) + + +@auth.command('set-role') +@click.argument('user_id', type=int) +@click.argument('role', type=click.Choice(USER_ROLES)) +@click.pass_context +def auth_set_role(ctx: click.Context, user_id: int, role: str) -> None: + """Change a user's role (admin only). + + You cannot change your own role: demoting the last admin here would leave + nobody able to undo it, so the API answers 403 (exit code 6). + """ + send_and_render(ctx, 'PATCH', f'/users/{user_id}', {'role': role}) diff --git a/sp_cli/commands/category.py b/sp_cli/commands/category.py new file mode 100644 index 0000000..09b58fc --- /dev/null +++ b/sp_cli/commands/category.py @@ -0,0 +1,83 @@ +"""``sp category`` — list and maintain the categories regression tests are filed under.""" + +from typing import Optional + +import click + +from sp_cli.constants import DESCRIPTION_MAX_LENGTH, NAME_MAX_LENGTH +from sp_cli.runner import clean_params, fetch_and_render, send_and_render + + +@click.group() +def category() -> None: + """List and maintain regression-test categories.""" + + +@category.command('ls') +@click.option('--limit', type=int, default=None, help='Page size (max 100).') +@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.pass_context +def category_ls(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: + """List categories alphabetically, each with how many tests reference it.""" + fetch_and_render(ctx, '/categories', clean_params({'limit': limit, 'offset': offset})) + + +@category.command('create') +@click.argument('name') +@click.option('--description', default=None, + help=f'Free-text description (max {DESCRIPTION_MAX_LENGTH} chars).') +@click.pass_context +def category_create(ctx: click.Context, name: str, description: Optional[str]) -> None: + """Create a category. Names are unique, so a duplicate is a 409 (exit code 8).""" + _check_widths(name, description) + send_and_render(ctx, 'POST', '/categories', + clean_params({'name': name, 'description': description})) + + +@category.command('edit') +@click.argument('category_id', type=int) +@click.option('--name', default=None, help=f'New name (max {NAME_MAX_LENGTH} chars).') +@click.option('--description', default=None, help='New description.') +@click.pass_context +def category_edit(ctx: click.Context, category_id: int, name: Optional[str], + description: Optional[str]) -> None: + """Rename a category or change its description. Only the fields you pass change.""" + _check_widths(name, description) + body = clean_params({'name': name, 'description': description}) + if not body: + raise click.UsageError('Nothing to update: pass --name or --description.') + send_and_render(ctx, 'PATCH', f'/categories/{category_id}', body) + + +@category.command('rm') +@click.argument('category_id', type=int) +@click.option('--yes', is_flag=True, default=False, help='Skip the confirmation prompt.') +@click.pass_context +def category_rm(ctx: click.Context, category_id: int, yes: bool) -> None: + """Delete a category no regression test references. + + The API refuses (409, exit code 8) while tests still point at it, since + dropping it would silently change which tests a suite selection picks up. + Detach them first with `sp regression edit --category ...`. + """ + if not yes: + click.confirm(f'Delete category {category_id}?', abort=True) + send_and_render(ctx, 'DELETE', f'/categories/{category_id}') + + +def _check_widths(name: Optional[str], description: Optional[str]) -> None: + """ + Reject over-long names and descriptions before the request goes out. + + :param name: The proposed category name, if any. + :type name: Optional[str] + :param description: The proposed description, if any. + :type description: Optional[str] + :raises click.BadParameter: when either exceeds the API's column width. + """ + if name is not None and not 1 <= len(name) <= NAME_MAX_LENGTH: + raise click.BadParameter( + f'must be 1 to {NAME_MAX_LENGTH} characters', param_hint='--name') + if description is not None and len(description) > DESCRIPTION_MAX_LENGTH: + raise click.BadParameter( + f'must be {DESCRIPTION_MAX_LENGTH} characters or less', param_hint='--description') diff --git a/sp_cli/commands/regression.py b/sp_cli/commands/regression.py index ec006d7..b0a11dc 100644 --- a/sp_cli/commands/regression.py +++ b/sp_cli/commands/regression.py @@ -1,15 +1,18 @@ -"""``sp regression`` — list regression-test definitions.""" +"""``sp regression`` — list, inspect, and maintain regression-test definitions.""" -from typing import Optional +from typing import Optional, Tuple import click -from sp_cli.runner import clean_params, fetch_and_render +from sp_cli.constants import (COMMAND_MAX_LENGTH, DESCRIPTION_MAX_LENGTH, + EXPECTED_RC_MAX, EXPECTED_RC_MIN, INPUT_TYPES, + OUTPUT_TYPES) +from sp_cli.runner import clean_params, fetch_and_render, send_and_render @click.group() def regression() -> None: - """List regression-test definitions.""" + """List, inspect, and maintain regression-test definitions.""" @regression.command('ls') @@ -33,3 +36,116 @@ def regression_ls(ctx: click.Context, category: Optional[str], tag: Optional[str params = clean_params({'category': category, 'tag': tag, 'active': active, 'sample_id': sample_id, 'limit': limit, 'offset': offset}) fetch_and_render(ctx, '/regression-tests', params) + + +@regression.command('show') +@click.argument('regression_test_id', type=int) +@click.pass_context +def regression_show(ctx: click.Context, regression_test_id: int) -> None: + """Show one regression test, including its expected outputs.""" + fetch_and_render(ctx, f'/regression-tests/{regression_test_id}') + + +@regression.command('create') +@click.option('--sample-id', type=int, required=True, help='Media sample this test runs against.') +@click.option('--command', required=True, + help=f'CCExtractor arguments to run (max {COMMAND_MAX_LENGTH} chars).') +@click.option('--category', 'categories', multiple=True, required=True, + help='Category name to file the test under (repeatable, at least one).') +@click.option('--input-type', type=click.Choice(INPUT_TYPES), default=None, + help='How the sample is fed in. The API defaults to file.') +@click.option('--output-type', type=click.Choice(OUTPUT_TYPES), default=None, + help='What the test produces. The API defaults to file.') +@click.option('--expected-rc', type=click.IntRange(EXPECTED_RC_MIN, EXPECTED_RC_MAX), default=None, + help='Expected process exit code. The API defaults to 0.') +@click.option('--description', default=None, + help=f'Free-text description (max {DESCRIPTION_MAX_LENGTH} chars).') +@click.option('--active', is_flag=True, default=False, + help='Join the CI suite immediately. Off by default, matching the API.') +@click.pass_context +def regression_create(ctx: click.Context, sample_id: int, command: str, + categories: Tuple[str, ...], input_type: Optional[str], + output_type: Optional[str], expected_rc: Optional[int], + description: Optional[str], active: bool) -> None: + """Create a regression test. + + A new test is created inactive unless --active is passed: the API's default + is that a maintainer should see what it actually produces on a verification + run before it joins the suite. + + Categories are given by name, not id, and every name must already exist -- + an unknown one is rejected rather than created implicitly. + """ + if not 1 <= len(command) <= COMMAND_MAX_LENGTH: + raise click.BadParameter( + f'must be 1 to {COMMAND_MAX_LENGTH} characters', param_hint='--command') + if description is not None and len(description) > DESCRIPTION_MAX_LENGTH: + raise click.BadParameter( + f'must be {DESCRIPTION_MAX_LENGTH} characters or less', param_hint='--description') + + body = clean_params({ + 'sample_id': sample_id, 'command': command, 'categories': list(categories), + 'input_type': input_type, 'output_type': output_type, + 'expected_rc': expected_rc, 'description': description, + }) + # Sent only when set: the API's load_default is False, so echoing that back + # would be indistinguishable from asking for it. + if active: + body['active'] = True + send_and_render(ctx, 'POST', '/regression-tests', body) + + +@regression.command('edit') +@click.argument('regression_test_id', type=int) +@click.option('--command', default=None, help=f'New command (max {COMMAND_MAX_LENGTH} chars).') +@click.option('--category', 'categories', multiple=True, + help='Replace the category list (repeatable). Categories are replaced, not merged.') +@click.option('--input-type', type=click.Choice(INPUT_TYPES), default=None, help='New input type.') +@click.option('--output-type', type=click.Choice(OUTPUT_TYPES), default=None, help='New output type.') +@click.option('--expected-rc', type=click.IntRange(EXPECTED_RC_MIN, EXPECTED_RC_MAX), default=None, + help='New expected exit code.') +@click.option('--description', default=None, help='New description.') +@click.option('--active/--inactive', 'active', default=None, + help='Add the test to the CI suite or retire it.') +@click.pass_context +def regression_edit(ctx: click.Context, regression_test_id: int, command: Optional[str], + categories: Tuple[str, ...], input_type: Optional[str], + output_type: Optional[str], expected_rc: Optional[int], + description: Optional[str], active: Optional[bool]) -> None: + """Update a regression test. Only the fields you pass are changed. + + --inactive is how you retire a test that has already run: it keeps the + history that `sp regression rm` would refuse to erase. + """ + if command is not None and not 1 <= len(command) <= COMMAND_MAX_LENGTH: + raise click.BadParameter( + f'must be 1 to {COMMAND_MAX_LENGTH} characters', param_hint='--command') + if description is not None and len(description) > DESCRIPTION_MAX_LENGTH: + raise click.BadParameter( + f'must be {DESCRIPTION_MAX_LENGTH} characters or less', param_hint='--description') + + body = clean_params({ + 'command': command, 'input_type': input_type, 'output_type': output_type, + 'expected_rc': expected_rc, 'description': description, 'active': active, + }) + if categories: + body['categories'] = list(categories) + if not body: + raise click.UsageError('Nothing to update: pass at least one field to change.') + send_and_render(ctx, 'PATCH', f'/regression-tests/{regression_test_id}', body) + + +@regression.command('rm') +@click.argument('regression_test_id', type=int) +@click.option('--yes', is_flag=True, default=False, help='Skip the confirmation prompt.') +@click.pass_context +def regression_rm(ctx: click.Context, regression_test_id: int, yes: bool) -> None: + """Delete a regression test that has never run. + + The API refuses (409, exit code 8) once any result references the test, + because deleting it would erase evidence of past regressions. Retire such a + test with `sp regression edit --inactive` instead. + """ + if not yes: + click.confirm(f'Delete regression test {regression_test_id}?', abort=True) + send_and_render(ctx, 'DELETE', f'/regression-tests/{regression_test_id}') diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index cf89480..45cb878 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -6,12 +6,14 @@ from sp_cli.client import ApiError from sp_cli.constants import (ARTIFACT_TYPES, CANCEL_REASON_MIN_LENGTH, - ERROR_GROUP_BY, ERROR_SEVERITIES, ERROR_TYPES, - INFRA_ERROR_TYPES, LOG_CONTAINS_MAX_LENGTH, - LOG_LEVELS, LOG_MAX_LIMIT, LOG_SOURCES, - PLATFORMS, RUN_STATUSES, SAMPLE_STATUSES) + COMMIT_SHA_LENGTH, ERROR_GROUP_BY, + ERROR_SEVERITIES, ERROR_TYPES, INFRA_ERROR_TYPES, + LOG_CONTAINS_MAX_LENGTH, LOG_LEVELS, + LOG_MAX_LIMIT, LOG_SOURCES, + MAX_REGRESSION_TEST_IDS, PLATFORMS, RUN_STATUSES, + SAMPLE_STATUSES) from sp_cli.output import render, render_error -from sp_cli.runner import clean_params, fetch_and_render +from sp_cli.runner import clean_params, fetch_and_render, send_and_render from sp_cli.triage import classify_sample, is_failure @@ -52,6 +54,44 @@ def run_ls(ctx: click.Context, status: Optional[str], platform: Optional[str], b fetch_and_render(ctx, '/runs', params) +@run.command('create') +@click.option('--commit', 'commit_sha', required=True, + help=f'Full {COMMIT_SHA_LENGTH}-char commit SHA. Short SHAs are rejected.') +@click.option('--platform', type=click.Choice(PLATFORMS), required=True, help='Test platform.') +@click.option('--repository', required=True, help='Fork to test, as owner/repo.') +@click.option('--branch', default=None, + help='Branch name. The API defaults to master when omitted.') +@click.option('--pull-request', 'pull_request', type=int, default=None, + help='Associate the run with a pull request number.') +@click.option('--test', 'regression_test_ids', type=int, multiple=True, + help=f'Restrict to these regression test ids (repeatable, max {MAX_REGRESSION_TEST_IDS}). ' + 'Omit to run the full active suite.') +@click.pass_context +def run_create(ctx: click.Context, commit_sha: str, platform: str, repository: str, + branch: Optional[str], pull_request: Optional[int], + regression_test_ids: Tuple[int, ...]) -> None: + """Queue a new CI run. + + The commit must already have a CI artifact built for that platform, so this + schedules a test of an existing build rather than triggering a compile. + """ + if len(commit_sha) != COMMIT_SHA_LENGTH or not all(c in '0123456789abcdefABCDEF' for c in commit_sha): + raise click.BadParameter( + f'must be a {COMMIT_SHA_LENGTH}-character hex string', param_hint='--commit') + if '/' not in repository.strip('/') or repository.count('/') != 1: + raise click.BadParameter('must be in owner/repo format', param_hint='--repository') + if len(regression_test_ids) > MAX_REGRESSION_TEST_IDS: + raise click.BadParameter( + f'at most {MAX_REGRESSION_TEST_IDS} ids', param_hint='--test') + + body = clean_params({ + 'commit_sha': commit_sha, 'platform': platform, 'repository': repository, + 'branch': branch, 'pull_request': pull_request, + 'regression_test_ids': list(regression_test_ids) or None, + }) + send_and_render(ctx, 'POST', '/runs', body) + + @run.command('show') @click.argument('run_id', type=int) @click.pass_context diff --git a/sp_cli/commands/sample.py b/sp_cli/commands/sample.py index cca7b1a..1dfb592 100644 --- a/sp_cli/commands/sample.py +++ b/sp_cli/commands/sample.py @@ -41,6 +41,19 @@ def sample_show(ctx: click.Context, sample_id: int) -> None: fetch_and_render(ctx, f'/samples/{sample_id}') +@sample.command('details') +@click.argument('sample_id', type=int) +@click.pass_context +def sample_details(ctx: click.Context, sample_id: int) -> None: + """Show everything known about a sample: upload record, extra files, media info. + + Media info is best-effort -- missing or unparseable XML reports null rather + than failing the whole response. Unlike the web page this never regenerates + that XML, because a GET should not write to the sample repository. + """ + fetch_and_render(ctx, f'/samples/{sample_id}/details') + + @sample.command('history') @click.argument('sample_id', type=int) @click.option('--platform', type=click.Choice(PLATFORMS), default=None, help='Test platform.') diff --git a/sp_cli/main.py b/sp_cli/main.py index 47d8c14..87bbcb4 100644 --- a/sp_cli/main.py +++ b/sp_cli/main.py @@ -6,7 +6,9 @@ from sp_cli import __version__ from sp_cli.client import ApiClient +from sp_cli.commands.admin import admin from sp_cli.commands.auth import auth +from sp_cli.commands.category import category from sp_cli.commands.investigate import investigate from sp_cli.commands.regression import regression from sp_cli.commands.run import run @@ -46,6 +48,8 @@ def cli(ctx: click.Context, base_url: str, token: Optional[str], output: str, ti cli.add_command(run) cli.add_command(sample) cli.add_command(regression) +cli.add_command(category) cli.add_command(auth) +cli.add_command(admin) cli.add_command(health) cli.add_command(queue) diff --git a/sp_cli/runner.py b/sp_cli/runner.py index 5276772..4d0f777 100644 --- a/sp_cli/runner.py +++ b/sp_cli/runner.py @@ -29,6 +29,32 @@ def fetch_and_render(ctx: click.Context, path: str, params: Optional[Dict[str, A render(payload, output) +def send_and_render(ctx: click.Context, method: str, path: str, + json_body: Optional[Dict[str, Any]] = None) -> None: + """ + Send a write request via the configured client and render it, exiting on error. + + The write-side twin of :func:`fetch_and_render`. + + :param ctx: The active Click context (carries the client and output mode). + :type ctx: click.Context + :param method: HTTP method (``POST``, ``PATCH``, ``DELETE``). + :type method: str + :param path: API path below ``/api/v1``. + :type path: str + :param json_body: Optional JSON request body. + :type json_body: Optional[Dict[str, Any]] + """ + client = ctx.obj['client'] + output = ctx.obj['output'] + try: + payload = client.request(method, path, json_body=json_body) + except ApiError as error: + render_error(error, output) + raise SystemExit(error.exit_code) + render(payload, output) + + def clean_params(params: Dict[str, Any]) -> Dict[str, Any]: """ Drop ``None`` values so unset options are not sent as query parameters. diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ac5e51..951394f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -682,3 +682,261 @@ def test_missing_log_file_is_distinguishable_from_a_missing_run(self, mock_get): envelope = json.loads(result.stderr) self.assertEqual(envelope['error']['code'], 'log_not_found') self.assertIn('artifacts', envelope['error']['details']['action_required']) + + +class WriteEndpointTests(unittest.TestCase): + """Cover the write and admin endpoints, pinned to the merged request schemas.""" + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_run_create_posts_the_full_body(self, mock_request): + """`run create` maps its options onto RunCreateRequestSchema field names.""" + mock_request.return_value = {'run_id': 9300, 'status': 'queued'} + sha = 'a' * 40 + result = self.runner.invoke(cli, [ + 'run', 'create', '--commit', sha, '--platform', 'linux', + '--repository', 'CCExtractor/ccextractor', '--branch', 'feature/x', + '--pull-request', '42', '--test', '18', '--test', '137']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with('POST', '/runs', json_body={ + 'commit_sha': sha, 'platform': 'linux', + 'repository': 'CCExtractor/ccextractor', 'branch': 'feature/x', + 'pull_request': 42, 'regression_test_ids': [18, 137]}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_run_create_rejects_a_short_sha_locally(self, mock_request): + """commit_sha is validated as 40 hex chars, so a short SHA never leaves the machine.""" + for bad in ('e6cd34e', 'z' * 40, 'a' * 39): + result = self.runner.invoke(cli, [ + 'run', 'create', '--commit', bad, '--platform', 'linux', + '--repository', 'CCExtractor/ccextractor']) + self.assertNotEqual(result.exit_code, 0, f'{bad!r} should be rejected') + mock_request.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_run_create_rejects_a_malformed_repository(self, mock_request): + """repository must be owner/repo, not a bare name or a URL.""" + for bad in ('ccextractor', 'https://github.com/CCExtractor/ccextractor'): + result = self.runner.invoke(cli, [ + 'run', 'create', '--commit', 'a' * 40, '--platform', 'linux', + '--repository', bad]) + self.assertNotEqual(result.exit_code, 0, f'{bad!r} should be rejected') + mock_request.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_regression_create_sends_categories_by_name(self, mock_request): + """Categories are given by name and are required; active defaults off, like the API.""" + mock_request.return_value = {'id': 5} + result = self.runner.invoke(cli, [ + 'regression', 'create', '--sample-id', '42', '--command', '-autoprogram', + '--category', 'DVB', '--category', 'General']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with('POST', '/regression-tests', json_body={ + 'sample_id': 42, 'command': '-autoprogram', 'categories': ['DVB', 'General']}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_regression_create_sends_active_only_when_asked(self, mock_request): + """The API's load_default is False, so a bare create must not send active at all.""" + mock_request.return_value = {'id': 5} + result = self.runner.invoke(cli, [ + 'regression', 'create', '--sample-id', '42', '--command', 'x', + '--category', 'DVB', '--active']) + + self.assertEqual(result.exit_code, 0) + self.assertTrue(mock_request.call_args.kwargs['json_body']['active']) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_regression_edit_requires_at_least_one_field(self, mock_request): + """An empty PATCH body is a usage error, not a pointless round trip.""" + result = self.runner.invoke(cli, ['regression', 'edit', '18']) + self.assertNotEqual(result.exit_code, 0) + mock_request.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_regression_edit_patches_only_given_fields(self, mock_request): + """PATCH is sparse: untouched options must not appear in the body.""" + mock_request.return_value = {'id': 18} + result = self.runner.invoke(cli, ['regression', 'edit', '18', '--inactive']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with( + 'PATCH', '/regression-tests/18', json_body={'active': False}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_regression_rm_confirms_before_deleting(self, mock_request): + """Deletion prompts unless --yes; declining must not issue the request.""" + result = self.runner.invoke(cli, ['regression', 'rm', '18'], input='n\n') + self.assertNotEqual(result.exit_code, 0) + mock_request.assert_not_called() + + mock_request.return_value = {'id': 18, 'deleted': True} + result = self.runner.invoke(cli, ['regression', 'rm', '18', '--yes']) + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with('DELETE', '/regression-tests/18', json_body=None) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_a_refused_delete_maps_to_its_own_exit_code(self, mock_request): + """409 means the world disagreed, not that the body was wrong -- exit 8, not 5.""" + mock_request.side_effect = ApiError( + 'conflict', 'Regression test 18 has 12 historical result(s).', 409, + {'result_count': 12}) + result = self.runner.invoke(cli, ['regression', 'rm', '18', '--yes']) + + self.assertEqual(result.exit_code, 8) + self.assertEqual(json.loads(result.stderr)['error']['code'], 'conflict') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_category_ls(self, mock_get): + """`category ls` reaches /categories with pagination.""" + mock_get.return_value = {'data': [], 'pagination': {}} + result = self.runner.invoke(cli, ['category', 'ls', '--limit', '10']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/categories', params={'limit': 10}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_category_create_and_edit(self, mock_request): + """Create takes a positional name; edit is sparse and needs a field.""" + mock_request.return_value = {'id': 3, 'name': 'DVB'} + result = self.runner.invoke(cli, ['category', 'create', 'DVB', '--description', 'DVB subs']) + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with( + 'POST', '/categories', json_body={'name': 'DVB', 'description': 'DVB subs'}) + + self.assertNotEqual(self.runner.invoke(cli, ['category', 'edit', '3']).exit_code, 0) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_category_name_width_is_enforced_locally(self, mock_request): + """The name column is 64 chars; a longer one is rejected before the request.""" + result = self.runner.invoke(cli, ['category', 'create', 'x' * 65]) + self.assertNotEqual(result.exit_code, 0) + mock_request.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_sample_details(self, mock_get): + """`sample details` is a distinct endpoint from `sample show`.""" + mock_get.return_value = {'sample_id': 42, 'media_info': None} + result = self.runner.invoke(cli, ['sample', 'details', '42']) + + self.assertEqual(result.exit_code, 0) + mock_get.assert_called_once_with('/samples/42/details', params=None) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_auth_whoami_and_users(self, mock_get): + """whoami hits /auth/me; users hits /users.""" + mock_get.return_value = {'user_id': 1, 'role': 'admin'} + self.assertEqual(self.runner.invoke(cli, ['auth', 'whoami']).exit_code, 0) + mock_get.assert_called_with('/auth/me', params=None) + + mock_get.return_value = {'data': [], 'pagination': {}} + self.assertEqual(self.runner.invoke(cli, ['auth', 'users']).exit_code, 0) + mock_get.assert_called_with('/users', params={}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_auth_set_role_is_restricted_to_the_role_enum(self, mock_request): + """PATCH /users/{id} validates against Role, so reject anything else locally.""" + self.assertNotEqual( + self.runner.invoke(cli, ['auth', 'set-role', '5', 'superuser']).exit_code, 0) + mock_request.assert_not_called() + + mock_request.return_value = {'user_id': 5, 'role': 'contributor'} + result = self.runner.invoke(cli, ['auth', 'set-role', '5', 'contributor']) + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with( + 'PATCH', '/users/5', json_body={'role': 'contributor'}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_admin_pause_and_resume_send_the_disabled_flag(self, mock_request): + """Pause and resume are the same PATCH with opposite booleans.""" + mock_request.return_value = {'platform': 'linux', 'disabled': True} + self.assertEqual(self.runner.invoke(cli, ['admin', 'pause', 'linux']).exit_code, 0) + mock_request.assert_called_with( + 'PATCH', '/system/maintenance/linux', json_body={'disabled': True}) + + self.assertEqual(self.runner.invoke(cli, ['admin', 'resume', 'linux']).exit_code, 0) + mock_request.assert_called_with( + 'PATCH', '/system/maintenance/linux', json_body={'disabled': False}) + + self.assertNotEqual(self.runner.invoke(cli, ['admin', 'pause', 'bsd']).exit_code, 0) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_blocked_user_add_takes_the_numeric_github_id(self, mock_request): + """The API keys on the numeric id, since logins can be changed and reused.""" + mock_request.return_value = {'user_id': 1234, 'comment': 'spam'} + result = self.runner.invoke(cli, ['admin', 'blocked-users', 'add', '1234', + '--comment', 'spam']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with( + 'POST', '/system/blocked-users', json_body={'user_id': 1234, 'comment': 'spam'}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_forbidden_extension_is_normalized_before_sending(self, mock_request): + """Stored without a leading dot and lower-cased, so normalize on the way out.""" + mock_request.return_value = {'extension': 'mkv'} + result = self.runner.invoke(cli, ['admin', 'forbidden-extensions', 'add', '.MKV']) + + self.assertEqual(result.exit_code, 0) + mock_request.assert_called_once_with( + 'POST', '/system/forbidden-extensions', json_body={'extension': 'mkv'}) + + self.assertNotEqual( + self.runner.invoke(cli, ['admin', 'forbidden-extensions', 'add', 'mk*v']).exit_code, 0) + + +class ScopeContractTests(unittest.TestCase): + """Pin the token scope list, which drifted once and broke every admin write. + + `system:write` was missing from TOKEN_SCOPES, so Click rejected + `--scope system:write` as an invalid choice and no CLI-created token could + ever authorize `sp admin pause` / `blocked-users` / `forbidden-extensions`. + Mocked tests could not catch it; only a live call did. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + def test_every_valid_scope_is_offered(self): + """The list must match mod_api.models.api_token.VALID_SCOPES exactly.""" + from sp_cli.constants import TOKEN_MAX_SCOPES, TOKEN_SCOPES + + self.assertEqual(set(TOKEN_SCOPES), { + 'runs:read', 'runs:write', 'results:read', 'baselines:write', + 'system:read', 'system:write', 'tokens:manage', + }) + # The API validates Length(max=len(VALID_SCOPES)), so asking for + # everything you are allowed must never fail validation. + self.assertEqual(TOKEN_MAX_SCOPES, len(TOKEN_SCOPES)) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_system_write_is_accepted_by_login(self, mock_request): + """Without this scope every `sp admin` write command 403s.""" + mock_request.return_value = {'token': 'x', 'scopes': ['system:write']} + result = self.runner.invoke(cli, [ + 'auth', 'login', '--email', 'a@b.c', '--password', 'pw', + '--scope', 'system:write']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(mock_request.call_args.kwargs['json_body']['scopes'], + ['system:write']) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_asking_for_every_scope_at_once_is_allowed(self, mock_request): + """The cap is len(VALID_SCOPES), so a full-access token is requestable.""" + from sp_cli.constants import TOKEN_SCOPES + + mock_request.return_value = {'token': 'x', 'scopes': list(TOKEN_SCOPES)} + args = ['auth', 'login', '--email', 'a@b.c', '--password', 'pw'] + for scope in TOKEN_SCOPES: + args += ['--scope', scope] + result = self.runner.invoke(cli, args) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(len(mock_request.call_args.kwargs['json_body']['scopes']), + len(TOKEN_SCOPES)) From 62ce37d6096246446cc305d96ad2dbc076ce7b7a Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 5 Aug 2026 23:33:13 +0530 Subject: [PATCH 06/10] feat: saved login session, colorized codes, progress spinner Three human-facing conveniences, all bound by the same rule: machine output stays clean. Every one of them is suppressed when the output is not a live terminal, and none of them can reach stdout in JSON mode. Saved session (sp_cli/config.py) `sp auth login` writes the token to ~/.config/sp/config.json so it does not have to be pasted into SP_API_TOKEN for every shell. Precedence is --token > SP_API_TOKEN > the file, so an explicit credential always wins. --no-save opts out for shared machines. The file is created through os.open at mode 0600 rather than chmod-ed afterwards, so the token is never briefly world-readable, and a warning is printed if an existing file has looser permissions. A corrupt file degrades to "logged out" instead of breaking every command, and `sp auth logout` clears it even when the server call fails -- an already-expired token must not be left behind on disk. Colour (sp_cli/output.py) The code and verdict columns are colorized by severity. Padding is applied before styling, because escape codes have no display width and colorizing first pushes every later column out of line. Gated on table mode, a TTY, NO_COLOR, and --no-color. Spinner (sp_cli/progress.py) Shown during the multi-page calls -- run failures, run logs --all, investigate -- and drawn on stderr, self-erasing, so even when it does run it cannot contaminate a payload being parsed on stdout. `sp shell`, the fourth item on the backlog, is dropped rather than built: a REPL is a second interface every future command must work in, it loses the pipes that make the CLI useful, and the primary consumer is an agent driving one-shot commands. --- sp_cli/commands/auth.py | 28 +++- sp_cli/commands/investigate.py | 20 ++- sp_cli/commands/run.py | 14 +- sp_cli/config.py | 137 ++++++++++++++++++ sp_cli/main.py | 20 ++- sp_cli/output.py | 84 ++++++++++- sp_cli/progress.py | 96 +++++++++++++ sp_cli/runner.py | 4 +- tests/test_cli.py | 4 +- tests/test_ux.py | 253 +++++++++++++++++++++++++++++++++ 10 files changed, 631 insertions(+), 29 deletions(-) create mode 100644 sp_cli/config.py create mode 100644 sp_cli/progress.py create mode 100644 tests/test_ux.py diff --git a/sp_cli/commands/auth.py b/sp_cli/commands/auth.py index 7fab29c..5b5c722 100644 --- a/sp_cli/commands/auth.py +++ b/sp_cli/commands/auth.py @@ -4,6 +4,7 @@ import click +from sp_cli import config from sp_cli.client import ApiError from sp_cli.constants import (TOKEN_MAX_DAYS, TOKEN_MIN_DAYS, TOKEN_SCOPES, USER_ROLES) @@ -25,13 +26,19 @@ def auth() -> None: default=TOKEN_MAX_DAYS, show_default=True, help='Token lifetime in days.') @click.option('--scope', 'scopes', multiple=True, type=click.Choice(TOKEN_SCOPES), help='Grant a specific scope; repeatable. Omit for the server default set.') +@click.option('--save/--no-save', 'save', default=True, show_default=True, + help='Save the token to ~/.config/sp/config.json (mode 0600) for later commands.') @click.pass_context def auth_login(ctx: click.Context, email: str, password: str, token_name: str, - expires_in_days: int, scopes: Tuple[str, ...]) -> None: - """Create an API token; store the printed value in SP_API_TOKEN. + expires_in_days: int, scopes: Tuple[str, ...], save: bool) -> None: + """Create an API token and save it for subsequent commands. The plaintext token is returned exactly once, at creation. Later `sp auth - tokens` calls list metadata only, so capture it now or create a new one. + tokens` calls list metadata only, so it is saved to + ~/.config/sp/config.json (mode 0600) unless --no-save is given. + + Precedence when a command runs is --token > SP_API_TOKEN > this file, so an + explicit flag or env var still wins over a saved session. """ client = ctx.obj['client'] output = ctx.obj['output'] @@ -45,6 +52,11 @@ def auth_login(ctx: click.Context, email: str, password: str, token_name: str, render_error(error, output) raise SystemExit(error.exit_code) + token = result.get('token') if isinstance(result, dict) else None + if save and token: + path = config.save_token(token, ctx.obj.get('base_url')) + # To stderr so it never contaminates the JSON on stdout. + click.echo(f'Token saved to {path} (mode 0600).', err=True) render(result, output) @@ -80,15 +92,21 @@ def auth_revoke(ctx: click.Context, token_id: int) -> None: @auth.command('logout') @click.pass_context def auth_logout(ctx: click.Context) -> None: - """Revoke the current API token.""" + """Revoke the current API token and drop the saved session. + + The local file is cleared even if the server call fails, so a token that + was already revoked or expired cannot leave a stale credential on disk. + """ client = ctx.obj['client'] output = ctx.obj['output'] try: client.request('DELETE', '/auth/tokens/current') except ApiError as error: + config.clear_token() render_error(error, output) raise SystemExit(error.exit_code) - click.echo('Token revoked.') + cleared = config.clear_token() + click.echo('Token revoked.' + (' Saved session cleared.' if cleared else '')) @auth.command('whoami') diff --git a/sp_cli/commands/investigate.py b/sp_cli/commands/investigate.py index 07594ce..b2c7f3c 100644 --- a/sp_cli/commands/investigate.py +++ b/sp_cli/commands/investigate.py @@ -8,6 +8,7 @@ from sp_cli.history import (NEW_REGRESSION, classify_history, group_by_verdict, split_history, unknown_history) from sp_cli.output import render, render_error +from sp_cli.progress import Spinner from sp_cli.runner import clean_params from sp_cli.triage import classify_sample, group_by_code, is_failure @@ -45,9 +46,10 @@ def investigate(ctx: click.Context, run_id: int, with_history: bool, depth = history_depth if history_depth is not None else DEFAULT_HISTORY_DEPTH try: - run = client.get(f'/runs/{run_id}') - summary = client.get(f'/runs/{run_id}/summary') - samples = client.get_paginated(f'/runs/{run_id}/samples') + with Spinner(f'Investigating run {run_id}', output != 'json'): + run = client.get(f'/runs/{run_id}') + summary = client.get(f'/runs/{run_id}/summary') + samples = client.get_paginated(f'/runs/{run_id}/samples') except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) @@ -62,7 +64,8 @@ def investigate(ctx: click.Context, run_id: int, with_history: bool, if with_history: try: - _attach_history(client, failures, run_id, run.get('platform'), depth) + with Spinner('Fetching sample history', output != 'json'): + _attach_history(client, failures, run_id, run.get('platform'), depth) except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) @@ -71,7 +74,7 @@ def investigate(ctx: click.Context, run_id: int, with_history: bool, if output == 'json': render(report, 'json') else: - _print_digest(report, with_history) + _print_digest(report, with_history, ctx.obj.get('color', False)) def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, @@ -113,7 +116,8 @@ def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, failure['history'] = classify_history(current, prior[:depth]) -def _print_digest(report: Dict[str, Any], with_history: bool = False) -> None: +def _print_digest(report: Dict[str, Any], with_history: bool = False, + color: bool = False) -> None: """ Print a human-readable investigation digest. @@ -121,6 +125,8 @@ def _print_digest(report: Dict[str, Any], with_history: bool = False) -> None: :type report: Dict[str, Any] :param with_history: Whether history verdicts were collected. :type with_history: bool + :param color: Whether to colorize the classification columns. + :type color: bool """ run = report['run'] summary = report['summary'] @@ -147,7 +153,7 @@ def _print_digest(report: Dict[str, Any], with_history: bool = False) -> None: failures: List[Dict[str, Any]] = report['failures'] if failures: click.echo() - render({'data': [_flatten(f, with_history) for f in failures]}, 'table') + render({'data': [_flatten(f, with_history) for f in failures]}, 'table', color) _print_regressions(failures, with_history) diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index 45cb878..67715a7 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -13,6 +13,7 @@ MAX_REGRESSION_TEST_IDS, PLATFORMS, RUN_STATUSES, SAMPLE_STATUSES) from sp_cli.output import render, render_error +from sp_cli.progress import Spinner from sp_cli.runner import clean_params, fetch_and_render, send_and_render from sp_cli.triage import classify_sample, is_failure @@ -116,13 +117,15 @@ def run_failures(ctx: click.Context, run_id: int) -> None: client = ctx.obj['client'] output = ctx.obj['output'] try: - samples = client.get_paginated(f'/runs/{run_id}/samples') + with Spinner(f'Fetching results for run {run_id}', output != 'json'): + samples = client.get_paginated(f'/runs/{run_id}/samples') except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) rows = [classify_sample(s) for s in samples if is_failure(s)] - render({'data': rows, 'summary': {'failures': len(rows), 'of_total': len(samples)}}, output) + render({'data': rows, 'summary': {'failures': len(rows), 'of_total': len(samples)}}, + output, ctx.obj.get('color', False)) @run.command('results') @@ -186,7 +189,7 @@ def run_diff(ctx: click.Context, run_id: int, sample_id: int, regression_id: Opt render_error(error, output) raise SystemExit(error.exit_code) - render(diffs[0] if len(diffs) == 1 else {'data': diffs}, output) + render(diffs[0] if len(diffs) == 1 else {'data': diffs}, output, ctx.obj.get('color', False)) @run.command('approve-baseline') @@ -384,11 +387,12 @@ def run_logs(ctx: click.Context, run_id: int, level: Optional[str], source: Opti client = ctx.obj['client'] output = ctx.obj['output'] try: - lines = client.get_cursor_paginated(f'/runs/{run_id}/logs', params) + with Spinner(f'Reading log for run {run_id}', output != 'json'): + lines = client.get_cursor_paginated(f'/runs/{run_id}/logs', params) except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) - render({'data': lines, 'summary': {'lines': len(lines)}}, output) + render({'data': lines, 'summary': {'lines': len(lines)}}, output, ctx.obj.get('color', False)) @run.command('errors') diff --git a/sp_cli/config.py b/sp_cli/config.py new file mode 100644 index 0000000..9b51daa --- /dev/null +++ b/sp_cli/config.py @@ -0,0 +1,137 @@ +"""Saved login session at ``~/.config/sp/config.json``. + +Holds the token produced by ``sp auth login`` so it does not have to be pasted +into ``SP_API_TOKEN`` for every shell. The file is written ``0600`` because it +holds a bearer credential; a token in a world-readable file is a token leak. + +Precedence is ``--token`` > ``SP_API_TOKEN`` > this file, so an explicit flag +or an env var always wins over whatever was saved earlier. +""" + +import json +import os +import stat +from pathlib import Path +from typing import Any, Dict, Optional + +#: Octal mode for the config file and its directory: owner-only. +_FILE_MODE = 0o600 +_DIR_MODE = 0o700 + + +def config_path() -> Path: + """ + Locate the config file, honouring ``XDG_CONFIG_HOME``. + + :return: Absolute path to ``sp/config.json`` under the config home. + :rtype: Path + """ + xdg = os.environ.get('XDG_CONFIG_HOME') + base = Path(xdg) if xdg else Path.home() / '.config' + return base / 'sp' / 'config.json' + + +def load() -> Dict[str, Any]: + """ + Read the saved session, tolerating a missing or corrupt file. + + A malformed file returns empty rather than raising: a bad config should + degrade to "not logged in", never break every command. + + :return: The saved mapping, or an empty dict. + :rtype: Dict[str, Any] + """ + path = config_path() + try: + with path.open(encoding='utf-8') as handle: + data = json.load(handle) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def saved_token() -> Optional[str]: + """ + Return the saved bearer token, if one was stored. + + :return: The token, or ``None`` when no usable session is saved. + :rtype: Optional[str] + """ + token = load().get('token') + return token if isinstance(token, str) and token else None + + +def save_token(token: str, base_url: Optional[str] = None) -> Path: + """ + Persist a token (and optionally the base URL it belongs to) at mode 0600. + + The file is created with restrictive permissions from the outset rather + than chmod-ed afterwards, so the secret is never briefly world-readable. + + :param token: The plaintext bearer token to store. + :type token: str + :param base_url: The API root the token authenticates against. + :type base_url: Optional[str] + :return: The path written. + :rtype: Path + """ + path = config_path() + path.parent.mkdir(parents=True, exist_ok=True, mode=_DIR_MODE) + + data = load() + data['token'] = token + if base_url: + data['base_url'] = base_url + + # Open through os.open so the mode applies at creation time. An existing + # file keeps its inode, so re-chmod it too. + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _FILE_MODE) + with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: + json.dump(data, handle, indent=2) + handle.write('\n') + os.chmod(path, _FILE_MODE) + return path + + +def clear_token() -> bool: + """ + Remove the saved token, leaving any other settings in place. + + :return: ``True`` if a token was actually removed. + :rtype: bool + """ + path = config_path() + data = load() + if 'token' not in data: + return False + del data['token'] + + if not data: + try: + path.unlink() + except OSError: + return False + return True + + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _FILE_MODE) + with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: + json.dump(data, handle, indent=2) + handle.write('\n') + return True + + +def is_world_readable() -> bool: + """ + Report whether the config file is readable by anyone but its owner. + + Used to warn when a token file was created by an older version, or had its + permissions loosened by hand. + + :return: ``True`` when group or other bits are set. + :rtype: bool + """ + try: + mode = config_path().stat().st_mode + except OSError: + return False + return bool(mode & (stat.S_IRWXG | stat.S_IRWXO)) diff --git a/sp_cli/main.py b/sp_cli/main.py index 87bbcb4..e5f6c32 100644 --- a/sp_cli/main.py +++ b/sp_cli/main.py @@ -4,7 +4,7 @@ import click -from sp_cli import __version__ +from sp_cli import __version__, config from sp_cli.client import ApiClient from sp_cli.commands.admin import admin from sp_cli.commands.auth import auth @@ -22,22 +22,36 @@ @click.option('--base-url', envvar='SP_BASE_URL', default=DEFAULT_BASE_URL, show_default=True, help='API base URL incl. the /api/v1 prefix. Env: SP_BASE_URL.') @click.option('--token', envvar='SP_API_TOKEN', default=None, - help='Bearer token sent with each request. Env: SP_API_TOKEN.') + help='Bearer token sent with each request. Env: SP_API_TOKEN. ' + 'Falls back to the session saved by `sp auth login`.') @click.option('--output', '-o', type=click.Choice(['json', 'table']), default='json', show_default=True, help='Output format.') @click.option('--timeout', type=int, default=30, show_default=True, help='Per-request timeout (seconds).') +@click.option('--no-color', is_flag=True, default=False, + help='Never colorize table output. Also honours the NO_COLOR environment variable.') @click.version_option(__version__, prog_name='sp') @click.pass_context -def cli(ctx: click.Context, base_url: str, token: Optional[str], output: str, timeout: int) -> None: +def cli(ctx: click.Context, base_url: str, token: Optional[str], output: str, + timeout: int, no_color: bool) -> None: """AI-friendly CLI for the CCExtractor CI / Sample Platform. Emits JSON by default so it can be driven by agents and scripts. Point it at a running platform with --base-url or the SP_BASE_URL environment variable, and authenticate with a token via --token / SP_API_TOKEN (see `sp auth login`). """ + # Precedence: --token / SP_API_TOKEN (both bound to `token` by Click) beat + # the saved session, so an explicit credential always wins. + if token is None: + token = config.saved_token() + if token and config.is_world_readable(): + click.echo(f'Warning: {config.config_path()} is readable by other users; ' + 'run chmod 600 on it.', err=True) + ctx.obj = { 'client': ApiClient(base_url, token=token, timeout=timeout), 'output': output, + 'base_url': base_url, + 'color': output == 'table' and not no_color, } if ctx.invoked_subcommand is None: from sp_cli.banner import show_welcome diff --git a/sp_cli/output.py b/sp_cli/output.py index bf61a32..9ab416a 100644 --- a/sp_cli/output.py +++ b/sp_cli/output.py @@ -1,6 +1,8 @@ """Render API responses to the terminal as JSON (default) or a simple table.""" import json +import os +import sys from typing import Any, Dict, List import click @@ -10,8 +12,30 @@ #: Value types rendered as plain table columns; nested structures are skipped. _SCALAR = (str, int, float, bool, type(None)) - -def render(payload: Any, output: str) -> None: +#: Columns worth colorizing, and the colour each value gets. Severity reads +#: left to right: red is a crash, yellow a wrong result, cyan a diff to review. +_CODE_COLORS = { + 'SEGFAULT': 'red', + 'ABORT': 'red', + 'TIMEOUT': 'yellow', + 'EXIT_CODE_MISMATCH': 'yellow', + 'MISSING_OUTPUT': 'magenta', + 'OUTPUT_DIFF': 'cyan', + 'PASS': 'green', + # `investigate --with-history` verdicts, shown in the same table. + 'NEW_REGRESSION': 'red', + 'STILL_FAILING': 'yellow', + 'NEVER_PASSED': 'magenta', + 'FLAKY': 'cyan', + 'NO_HISTORY': 'white', + 'UNKNOWN': 'white', +} + +#: Only these columns are ever colorized; everything else stays plain. +_COLORIZED_COLUMNS = ('code', 'verdict') + + +def render(payload: Any, output: str, color: bool = False) -> None: """ Render a successful API payload in the requested format. @@ -22,13 +46,17 @@ def render(payload: Any, output: str) -> None: :type payload: Any :param output: Either ``json`` or ``table``. :type output: str + :param color: Whether the caller wants colour; still suppressed when stdout + is not a TTY or ``NO_COLOR`` is set. + :type color: bool """ if output == 'json': click.echo(json.dumps(payload, indent=2)) return + use_color = _color_enabled(color) if isinstance(payload, dict) and isinstance(payload.get('data'), list): - _print_rows(payload['data']) + _print_rows(payload['data'], use_color) footer = _footer(payload) if footer: click.echo(f"\n{footer}") @@ -38,6 +66,25 @@ def render(payload: Any, output: str) -> None: click.echo(json.dumps(payload, indent=2)) +def _color_enabled(requested: bool) -> bool: + """ + Decide whether colour may actually be emitted. + + Colour is decoration, so it is dropped whenever the output is not a live + terminal -- piping into ``jq`` or a file must never receive escape codes. + + :param requested: Whether the caller asked for colour. + :type requested: bool + :return: ``True`` only when colour is both wanted and safe. + :rtype: bool + """ + if not requested: + return False + if os.environ.get('NO_COLOR'): + return False + return sys.stdout.isatty() + + def render_error(error: ApiError, output: str) -> None: """ Render an API error as a JSON envelope on stderr, regardless of output mode. @@ -80,12 +127,14 @@ def _footer(payload: Dict[str, Any]) -> str: return '' -def _print_rows(rows: List[Any]) -> None: +def _print_rows(rows: List[Any], color: bool = False) -> None: """ Print a list of flat dicts as an aligned table of their scalar fields. :param rows: The list of row dicts to render. :type rows: List[Any] + :param color: Whether to colorize the classification columns. + :type color: bool """ if not rows: click.echo('(no results)') @@ -108,7 +157,32 @@ def _print_rows(rows: List[Any]) -> None: click.echo(' '.join(col.ljust(widths[col]) for col in columns)) click.echo(' '.join('-' * widths[col] for col in columns)) for row in rows: - click.echo(' '.join(_cell(row.get(col)).ljust(widths[col]) for col in columns)) + # Padded first, then styled: escape codes have no display width, so + # colorizing before ljust would push every later column out of line. + click.echo(' '.join( + _paint(_cell(row.get(col)).ljust(widths[col]), col, row.get(col), color) + for col in columns)) + + +def _paint(padded: str, column: str, value: Any, color: bool) -> str: + """ + Apply the classification colour to an already-padded cell. + + :param padded: The cell text, already widened to the column width. + :type padded: str + :param column: The column name the cell belongs to. + :type column: str + :param value: The raw cell value, used to pick the colour. + :type value: Any + :param color: Whether colour is enabled at all. + :type color: bool + :return: The cell, styled or untouched. + :rtype: str + """ + if not color or column not in _COLORIZED_COLUMNS: + return padded + fg = _CODE_COLORS.get(str(value)) + return click.style(padded, fg=fg) if fg else padded def _print_kv(record: Dict[str, Any]) -> None: diff --git a/sp_cli/progress.py b/sp_cli/progress.py new file mode 100644 index 0000000..68a8680 --- /dev/null +++ b/sp_cli/progress.py @@ -0,0 +1,96 @@ +"""A minimal progress spinner for slow, multi-page calls. + +Decoration, so it obeys the same rule as colour: it is suppressed whenever the +output is not a live terminal, whenever ``NO_COLOR`` is set, and always in JSON +mode. It draws on **stderr**, so even when it does run it cannot contaminate the +payload a caller is parsing on stdout. +""" + +import itertools +import os +import sys +import threading +from types import TracebackType +from typing import Optional, Type + +#: Frames cycle at this interval, in seconds. +_INTERVAL = 0.1 + +#: Plain ASCII so it renders in a bare terminal without a Unicode font. +_FRAMES = ('|', '/', '-', '\\') + + +class Spinner: + """Context manager that animates a label on stderr while work runs.""" + + def __init__(self, label: str, enabled: bool = True) -> None: + """ + Configure the spinner. + + :param label: Text shown beside the animation. + :type label: str + :param enabled: Whether the caller wants it; still suppressed when + stderr is not a TTY or ``NO_COLOR`` is set. + :type enabled: bool + """ + self.label = label + self.enabled = enabled and _spinner_allowed() + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + + def __enter__(self) -> 'Spinner': + """ + Start animating, if enabled. + + :return: This spinner. + :rtype: Spinner + """ + if self.enabled: + self._thread = threading.Thread(target=self._spin, daemon=True) + self._thread.start() + return self + + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + traceback: Optional[TracebackType]) -> None: + """ + Stop animating and erase the line. + + :param exc_type: Exception class, if the block raised. + :type exc_type: Optional[Type[BaseException]] + :param exc: The exception, if the block raised. + :type exc: Optional[BaseException] + :param traceback: The traceback, if the block raised. + :type traceback: Optional[TracebackType] + """ + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + self._erase() + + def _spin(self) -> None: + """Draw frames until asked to stop.""" + for frame in itertools.cycle(_FRAMES): + if self._stop.is_set(): + return + sys.stderr.write(f'\r{frame} {self.label}') + sys.stderr.flush() + if self._stop.wait(_INTERVAL): + return + + def _erase(self) -> None: + """Blank the spinner line so it leaves no residue behind.""" + sys.stderr.write('\r' + ' ' * (len(self.label) + 2) + '\r') + sys.stderr.flush() + + +def _spinner_allowed() -> bool: + """ + Report whether a spinner may be drawn at all. + + :return: ``True`` only when stderr is an interactive terminal. + :rtype: bool + """ + if os.environ.get('NO_COLOR'): + return False + return sys.stderr.isatty() diff --git a/sp_cli/runner.py b/sp_cli/runner.py index 4d0f777..f061603 100644 --- a/sp_cli/runner.py +++ b/sp_cli/runner.py @@ -26,7 +26,7 @@ def fetch_and_render(ctx: click.Context, path: str, params: Optional[Dict[str, A except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) - render(payload, output) + render(payload, output, ctx.obj.get('color', False)) def send_and_render(ctx: click.Context, method: str, path: str, @@ -52,7 +52,7 @@ def send_and_render(ctx: click.Context, method: str, path: str, except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) - render(payload, output) + render(payload, output, ctx.obj.get('color', False)) def clean_params(params: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/test_cli.py b/tests/test_cli.py index 951394f..259f20f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -920,7 +920,7 @@ def test_system_write_is_accepted_by_login(self, mock_request): mock_request.return_value = {'token': 'x', 'scopes': ['system:write']} result = self.runner.invoke(cli, [ 'auth', 'login', '--email', 'a@b.c', '--password', 'pw', - '--scope', 'system:write']) + '--scope', 'system:write', '--no-save']) self.assertEqual(result.exit_code, 0) self.assertEqual(mock_request.call_args.kwargs['json_body']['scopes'], @@ -932,7 +932,7 @@ def test_asking_for_every_scope_at_once_is_allowed(self, mock_request): from sp_cli.constants import TOKEN_SCOPES mock_request.return_value = {'token': 'x', 'scopes': list(TOKEN_SCOPES)} - args = ['auth', 'login', '--email', 'a@b.c', '--password', 'pw'] + args = ['auth', 'login', '--email', 'a@b.c', '--password', 'pw', '--no-save'] for scope in TOKEN_SCOPES: args += ['--scope', scope] result = self.runner.invoke(cli, args) diff --git a/tests/test_ux.py b/tests/test_ux.py new file mode 100644 index 0000000..cc58e45 --- /dev/null +++ b/tests/test_ux.py @@ -0,0 +1,253 @@ +"""Tests for the saved session, colour, and spinner behaviour. + +The rule these all share: a human-only effect must never reach machine output. +""" + +import json +import os +import stat +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from click.testing import CliRunner + +from sp_cli import config +from sp_cli.main import cli +from sp_cli.output import render + +CLASSIFIED_ROWS = { + 'data': [ + {'sample_name': 'dvb', 'code': 'SEGFAULT', 'verdict': 'NEW_REGRESSION'}, + {'sample_name': 'ok', 'code': 'PASS', 'verdict': 'STILL_FAILING'}, + ], +} + + +class SavedSessionTests(unittest.TestCase): + """`sp auth login` persists a token; everything else reads it back.""" + + def setUp(self): + """Point the config module at a throwaway directory.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + patcher = mock.patch.dict(os.environ, {'XDG_CONFIG_HOME': self.tmp.name}) + patcher.start() + self.addCleanup(patcher.stop) + self.runner = CliRunner() + + def test_token_file_is_owner_only(self): + """The file holds a bearer credential, so it must not be group/world readable.""" + path = config.save_token('secret-token') + + mode = stat.S_IMODE(Path(path).stat().st_mode) + self.assertEqual(mode, 0o600, f'expected 0600, got {oct(mode)}') + self.assertFalse(config.is_world_readable()) + + def test_saved_token_round_trips(self): + """What login writes is what later commands read back.""" + config.save_token('secret-token', 'http://example.test/api/v1') + + self.assertEqual(config.saved_token(), 'secret-token') + self.assertEqual(config.load()['base_url'], 'http://example.test/api/v1') + + def test_a_corrupt_file_degrades_to_logged_out(self): + """A malformed config must mean 'no session', not break every command.""" + path = config.config_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('{not json', encoding='utf-8') + + self.assertEqual(config.load(), {}) + self.assertIsNone(config.saved_token()) + + def test_clear_token_removes_only_the_credential(self): + """Logging out drops the token but keeps unrelated settings.""" + config.save_token('secret-token', 'http://example.test/api/v1') + + self.assertTrue(config.clear_token()) + self.assertIsNone(config.saved_token()) + self.assertEqual(config.load().get('base_url'), 'http://example.test/api/v1') + + @mock.patch('sp_cli.client.ApiClient.__init__', return_value=None) + def test_explicit_token_outranks_the_saved_session(self, mock_init): + """--token beats the file, so an explicit credential always wins.""" + config.save_token('from-file') + with mock.patch('sp_cli.client.ApiClient.get', return_value={'status': 'ok'}): + self.runner.invoke(cli, ['--token', 'from-flag', 'health']) + + self.assertEqual(mock_init.call_args.kwargs['token'], 'from-flag') + + @mock.patch('sp_cli.client.ApiClient.__init__', return_value=None) + def test_env_var_outranks_the_saved_session(self, mock_init): + """SP_API_TOKEN beats the file too; only an unset env falls through.""" + config.save_token('from-file') + with mock.patch.dict(os.environ, {'SP_API_TOKEN': 'from-env'}): + with mock.patch('sp_cli.client.ApiClient.get', return_value={'status': 'ok'}): + self.runner.invoke(cli, ['health']) + + self.assertEqual(mock_init.call_args.kwargs['token'], 'from-env') + + @mock.patch('sp_cli.client.ApiClient.__init__', return_value=None) + def test_the_file_is_read_when_nothing_else_provides_a_token(self, mock_init): + """With no flag and no env var, the saved session is what reaches the client.""" + config.save_token('from-file') + with mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop('SP_API_TOKEN', None) + with mock.patch('sp_cli.client.ApiClient.get', return_value={'status': 'ok'}): + self.runner.invoke(cli, ['health']) + + self.assertEqual(mock_init.call_args.kwargs['token'], 'from-file') + + @mock.patch('sp_cli.client.ApiClient.request') + def test_login_saves_the_token_and_says_so_on_stderr(self, mock_request): + """The confirmation goes to stderr so it cannot contaminate the JSON payload.""" + mock_request.return_value = {'token': 'brand-new', 'token_id': 7} + result = self.runner.invoke(cli, [ + 'auth', 'login', '--email', 'a@b.c', '--password', 'pw']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(config.saved_token(), 'brand-new') + self.assertIn('Token saved', result.stderr) + # stdout stays pure JSON. + self.assertEqual(json.loads(result.stdout)['token'], 'brand-new') + + @mock.patch('sp_cli.client.ApiClient.request') + def test_no_save_leaves_nothing_on_disk(self, mock_request): + """--no-save is for shared machines: the token is printed but never written.""" + mock_request.return_value = {'token': 'brand-new'} + result = self.runner.invoke(cli, [ + 'auth', 'login', '--email', 'a@b.c', '--password', 'pw', '--no-save']) + + self.assertEqual(result.exit_code, 0) + self.assertIsNone(config.saved_token()) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_logout_clears_the_file_even_when_the_server_rejects_it(self, mock_request): + """An already-expired token must not be left behind on disk.""" + from sp_cli.client import ApiError + config.save_token('stale') + mock_request.side_effect = ApiError('unauthorized', 'Token expired.', 401) + + result = self.runner.invoke(cli, ['auth', 'logout']) + + self.assertEqual(result.exit_code, 6) + self.assertIsNone(config.saved_token()) + + +class ColorTests(unittest.TestCase): + """Colour is decoration: table mode, TTY only, never in JSON.""" + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + def test_no_escape_codes_when_stdout_is_not_a_tty(self): + """Piping into jq or a file must receive plain text.""" + runner = CliRunner() + with runner.isolation() as (out, _err, _): + with mock.patch('sys.stdout.isatty', return_value=False): + render(CLASSIFIED_ROWS, 'table', color=True) + written = out.getvalue().decode() + + self.assertNotIn('\x1b[', written) + + def test_escape_codes_appear_only_for_the_classification_columns(self): + """A TTY gets colour, but only on code/verdict -- names stay plain.""" + with mock.patch('sys.stdout.isatty', return_value=True), \ + mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop('NO_COLOR', None) + with mock.patch('click.echo') as mock_echo: + render(CLASSIFIED_ROWS, 'table', color=True) + + body = '\n'.join(str(call.args[0]) for call in mock_echo.call_args_list if call.args) + self.assertIn('\x1b[', body) + + def test_no_color_env_var_is_honoured(self): + """NO_COLOR is a cross-tool convention; respect it even on a TTY.""" + runner = CliRunner() + with runner.isolation() as (out, _err, _): + with mock.patch('sys.stdout.isatty', return_value=True), \ + mock.patch.dict(os.environ, {'NO_COLOR': '1'}): + render(CLASSIFIED_ROWS, 'table', color=True) + written = out.getvalue().decode() + + self.assertNotIn('\x1b[', written) + + def test_json_mode_is_never_colorized(self): + """JSON is the machine contract; colour must not reach it under any flag.""" + runner = CliRunner() + with runner.isolation() as (out, _err, _): + with mock.patch('sys.stdout.isatty', return_value=True): + render(CLASSIFIED_ROWS, 'json', color=True) + written = out.getvalue().decode() + + self.assertNotIn('\x1b[', written) + json.loads(written) + + @mock.patch('sp_cli.commands.run.fetch_and_render') + def test_no_color_flag_turns_the_context_flag_off(self, mock_fetch): + """--no-color must reach the context, not just avoid crashing.""" + captured = {} + mock_fetch.side_effect = lambda ctx, *a, **kw: captured.update(ctx.obj) + + self.runner.invoke(cli, ['-o', 'table', 'run', 'ls']) + self.assertTrue(captured['color'], 'table mode should enable colour by default') + + captured.clear() + self.runner.invoke(cli, ['--no-color', '-o', 'table', 'run', 'ls']) + self.assertFalse(captured['color']) + + captured.clear() + self.runner.invoke(cli, ['run', 'ls']) + self.assertFalse(captured['color'], 'JSON mode must never request colour') + + def test_colour_padding_keeps_columns_aligned(self): + """Escape codes have no display width, so they must sit outside the padding.""" + with mock.patch('sys.stdout.isatty', return_value=True), \ + mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop('NO_COLOR', None) + with mock.patch('click.echo') as mock_echo: + render(CLASSIFIED_ROWS, 'table', color=True) + + lines = [str(call.args[0]) for call in mock_echo.call_args_list if call.args] + # Strip escape sequences and confirm every row is the same visible width. + import re + plain = [re.sub(r'\x1b\[[0-9;]*m', '', line) for line in lines if line.strip()] + widths = {len(line) for line in plain} + self.assertEqual(len(widths), 1, f'ragged columns: {plain}') + + +class SpinnerTests(unittest.TestCase): + """The spinner draws on stderr and only when stderr is interactive.""" + + def test_disabled_when_stderr_is_not_a_tty(self): + """Redirected stderr means no animation frames at all.""" + from sp_cli.progress import Spinner + with mock.patch('sys.stderr.isatty', return_value=False): + spinner = Spinner('working', enabled=True) + self.assertFalse(spinner.enabled) + + def test_disabled_when_the_caller_says_so(self): + """JSON mode passes enabled=False, which wins even on a TTY.""" + from sp_cli.progress import Spinner + with mock.patch('sys.stderr.isatty', return_value=True): + spinner = Spinner('working', enabled=False) + self.assertFalse(spinner.enabled) + + def test_no_color_also_silences_the_spinner(self): + """NO_COLOR signals 'no decoration', which covers the spinner too.""" + from sp_cli.progress import Spinner + with mock.patch('sys.stderr.isatty', return_value=True), \ + mock.patch.dict(os.environ, {'NO_COLOR': '1'}): + spinner = Spinner('working', enabled=True) + self.assertFalse(spinner.enabled) + + def test_a_disabled_spinner_writes_nothing(self): + """The context manager must be a no-op when disabled, not a silent thread leak.""" + from sp_cli.progress import Spinner + with mock.patch('sys.stderr.isatty', return_value=False): + with mock.patch('sys.stderr.write') as mock_write: + with Spinner('working', enabled=True): + pass + mock_write.assert_not_called() From 34baa1756e3982ac75e44192e427ab600370e489 Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Wed, 5 Aug 2026 23:33:40 +0530 Subject: [PATCH 07/10] docs: document the full command surface and exit codes The README still showed only the handful of commands the first draft had, and described `sp run logs` as "raw run logs" from when it was a stub. Groups the commands by what you are actually doing -- investigating a failure, browsing, maintaining tests, administering -- rather than listing them flat, and documents `sp auth login` as the way to authenticate now that the token is saved. Adds the exit-code table. Scripts and agents branch on these, so they belong in the README rather than only in the client docstring. --- README.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 66e7063..0acef6b 100644 --- a/README.md +++ b/README.md @@ -26,17 +26,64 @@ export SP_API_TOKEN= # if the API Both can also be passed per-command with `--base-url` and `--token`. +Or log in once and let `sp` remember the token: + +```bash +sp auth login --email you@example.com +``` + +That writes the token to `~/.config/sp/config.json` with mode `0600`. Precedence +is `--token` > `SP_API_TOKEN` > the saved file, so an explicit credential always +wins. `sp auth logout` revokes it and clears the file; `--no-save` skips writing +it at all. + ## Usage +### Investigating a failure + ```bash -sp # banner / help -sp health # API + dependency health -sp run ls # list CI runs -sp run summary # pass/fail summary for a run -sp run failures # failing tests, each auto-classified -sp run diff # expected-vs-actual diff for a result -sp run logs # raw run logs -sp investigate # one-shot triage: info + counts + classified failures +sp investigate # one-shot triage: info + counts + classified failures +sp investigate --with-history # ... and whether each failure is new +sp run summary # pass/fail summary for a run +sp run failures # failing tests, each auto-classified +sp run error-summary # grouped error counts — cheapest first look +sp run errors # structured per-test errors +sp run infra-errors # VM / checkout / build / worker failures +sp run diff # expected-vs-actual diff for a result +sp run logs --level error # build log, cursor-paginated +sp run artifacts # binary, coredump, outputs, build log +``` + +### Running and browsing + +```bash +sp health # API + dependency health +sp queue # queue depth and running jobs +sp run ls # list CI runs +sp run create --commit --platform linux --repository owner/repo +sp sample ls / show / details # media samples +sp regression ls / show # regression-test definitions +sp category ls # categories, with test counts +``` + +### Maintaining tests (contributor or admin) + +```bash +sp regression create --sample-id 42 --command '-autoprogram' --category DVB +sp regression edit 18 --inactive # retire a test that already has history +sp regression rm 18 # only allowed if it has never run +sp category create DVB --description 'DVB subtitles' +``` + +### Administration (admin only) + +```bash +sp auth whoami # who this token is, and its role +sp auth users # list platform users +sp admin maintenance # is CI paused? +sp admin pause linux # stop dispatching to a platform +sp admin blocked-users add --comment 'spam' +sp admin forbidden-extensions add exe ``` Add `-o table` to any command for a human-readable view (default is JSON): @@ -45,12 +92,35 @@ Add `-o table` to any command for a human-readable view (default is JSON): sp -o table investigate 9299 ``` +In table mode on a terminal, the `code` and `verdict` columns are colorized. +Colour is dropped automatically when the output is piped, and can be turned off +with `--no-color` or the standard `NO_COLOR` environment variable — JSON output +is never colorized. + ### The classifier `sp` labels each failure with a stable code — `SEGFAULT`, `ABORT`, `TIMEOUT`, `EXIT_CODE_MISMATCH`, `MISSING_OUTPUT`, `OUTPUT_DIFF`, `PASS` — so a person or an agent gets a straight answer about *why* a test failed, without reading logs. +With `--with-history`, each failure also gets a verdict across previous runs: +`NEW_REGRESSION`, `STILL_FAILING`, `NEVER_PASSED`, `FLAKY`, `NO_HISTORY`. + +### Exit codes + +Scripts and agents can branch on the exit status: + +| Code | Meaning | +|------|---------| +| 0 | success | +| 1 | unspecified error | +| 3 | could not reach the API | +| 4 | not found | +| 5 | validation error | +| 6 | authentication / authorization failure | +| 7 | rate limited | +| 8 | conflict (e.g. deleting a test that has results) | + ## Development ```bash From c29619c52013cdcd3d2c98d5a2fa13dd26d34089 Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Thu, 6 Aug 2026 00:14:52 +0530 Subject: [PATCH 08/10] fix: retry transient GET failures, add run output --decode Both of these are things a live run against production surfaced that the mocked tests could not. Retry `investigate --with-history` makes one call per failing sample. On a real run -- 45 failures, ~1.5s per request -- that is a 60-90 second window, and a single 30s read timeout partway through aborted the whole command with exit 3, discarding every lookup already done. That is exactly what happened. Failed GETs are now retried with exponential backoff plus jitter, for connection errors, read timeouts, 429 and 5xx. Retry-After is honoured when the server sends it, but clamped to 30s so a header of 3600 cannot hang the CLI. Jitter is there so a fleet of agents riding out the same blip does not resynchronise into a second thundering herd. Only GET is retried. POST /runs is not idempotent -- a retry racing a slow-but-successful first attempt would queue the run twice -- and a repeated DELETE turns a success into a confusing 404. 4xx other than 429 is deterministic, so asking again just wastes a round trip. Notices go to stderr, never stdout, so JSON stays parseable. --retries 0 restores the previous fail-fast behaviour. --decode The output endpoint returns the file base64-encoded inside a JSON envelope, so `sp run output` printed a multi-kilobyte blob that no one can read and no diff tool can consume. --decode writes the decoded bytes to stdout instead: sp run output 9388 11 --decode > actual.srt Written to sys.stdout.buffer rather than echoed, because these are subtitle files carrying CRLF and sometimes non-UTF-8 bytes -- re-encoding them would corrupt the very diff you are trying to read. Verified against production: 5562 bytes out, CRLF intact. Two existing client tests now pass retries=0. They assert the exit-code mapping, not the backoff, and 429 is retryable -- leaving the default on made them sleep for seconds. --- README.md | 19 +++++++ sp_cli/client.py | 83 +++++++++++++++++++++++++-- sp_cli/commands/run.py | 48 +++++++++++++++- sp_cli/main.py | 6 +- tests/test_cli.py | 67 ++++++++++++++++++++++ tests/test_client.py | 124 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 336 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0acef6b..ef68e4a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,14 @@ sp run logs --level error # build log, cursor-paginated sp run artifacts # binary, coredump, outputs, build log ``` +To get the actual output file rather than the JSON envelope it arrives in: + +```bash +sp run output --decode > actual.srt +sp run output --side expected --decode > expected.srt +diff expected.srt actual.srt +``` + ### Running and browsing ```bash @@ -106,6 +114,17 @@ agent gets a straight answer about *why* a test failed, without reading logs. With `--with-history`, each failure also gets a verdict across previous runs: `NEW_REGRESSION`, `STILL_FAILING`, `NEVER_PASSED`, `FLAKY`, `NO_HISTORY`. +### Reliability + +Failed `GET`s are retried with exponential backoff — connection failures, read +timeouts, `429`, and `5xx`. This matters most for `investigate --with-history`, +which makes one call per failing sample: without it a single blip partway +through discards every lookup before it. Retry notices go to stderr, so JSON on +stdout stays clean. Tune with `--retries N`; `--retries 0` fails fast. + +Writes are never retried. `POST /runs` is not idempotent, and a retry that +raced a slow-but-successful first attempt would queue the run twice. + ### Exit codes Scripts and agents can branch on the exit status: diff --git a/sp_cli/client.py b/sp_cli/client.py index 3f8c6a7..f0a5be1 100644 --- a/sp_cli/client.py +++ b/sp_cli/client.py @@ -1,9 +1,29 @@ """HTTP client for the CCExtractor CI System API (`/api/v1`).""" +import random +import sys +import time from typing import Any, Dict, List, Optional import requests # type: ignore[import-untyped] +#: Statuses where the server said "not now" rather than "no". Retrying these is +#: the difference between riding out a blip and losing a whole investigation: +#: ``investigate --with-history`` makes one call per failing sample, so a single +#: timeout 40 lookups in would otherwise throw away everything before it. +_RETRY_STATUSES = frozenset({429, 502, 503, 504}) + +#: Only GET is retried. POST /runs is not idempotent -- a retry that raced a +#: slow-but-successful first attempt would queue the run twice -- and a repeated +#: DELETE turns a success into a confusing 404. +_RETRYABLE_METHODS = frozenset({'GET'}) + +#: Base for exponential backoff, in seconds: 0.5, 1.0, 2.0 ... +_BACKOFF_BASE = 0.5 + +#: Ceiling on a single sleep, so a large Retry-After cannot hang the CLI. +_BACKOFF_MAX = 30.0 + class ApiError(Exception): """Raised when an API request fails, carrying the structured error envelope.""" @@ -57,7 +77,8 @@ def exit_code(self) -> int: class ApiClient: """Minimal client over the JSON API. Sends a bearer token when configured.""" - def __init__(self, base_url: str, token: Optional[str] = None, timeout: int = 30) -> None: + def __init__(self, base_url: str, token: Optional[str] = None, timeout: int = 30, + retries: int = 2) -> None: """ Configure the client. @@ -67,10 +88,13 @@ def __init__(self, base_url: str, token: Optional[str] = None, timeout: int = 30 :type token: Optional[str] :param timeout: Per-request timeout in seconds. :type timeout: int + :param retries: Extra attempts for a failed GET. 0 disables retrying. + :type retries: int """ self.base_url = base_url.rstrip('/') self.token = token self.timeout = timeout + self.retries = max(0, retries) self.session = requests.Session() def _headers(self) -> Dict[str, str]: @@ -103,11 +127,25 @@ def request(self, method: str, path: str, params: Optional[Dict[str, Any]] = Non :rtype: Any """ url = f"{self.base_url}{path}" - try: - response = self.session.request(method, url, params=params, json=json_body, - headers=self._headers(), timeout=self.timeout) - except requests.RequestException as exc: - raise ApiError('connection_error', f'Could not reach {url}: {exc}') + attempts = 1 + if method.upper() in _RETRYABLE_METHODS: + attempts += self.retries + + for attempt in range(1, attempts + 1): + last = attempt == attempts + try: + response = self.session.request(method, url, params=params, json=json_body, + headers=self._headers(), timeout=self.timeout) + except requests.RequestException as exc: + if last: + raise ApiError('connection_error', f'Could not reach {url}: {exc}') + self._wait(attempt, None, f'{type(exc).__name__} on {method} {path}') + continue + + if response.status_code in _RETRY_STATUSES and not last: + self._wait(attempt, response, f'HTTP {response.status_code} on {method} {path}') + continue + break if response.status_code == 204: return None @@ -129,6 +167,39 @@ def request(self, method: str, path: str, params: Optional[Dict[str, Any]] = Non return payload + def _wait(self, attempt: int, response: Optional[Any], reason: str) -> None: + """ + Sleep before the next attempt, and say so on stderr. + + Backoff is exponential with jitter, so a fleet of agents retrying the + same blip does not resynchronise into a second thundering herd. A + ``Retry-After`` header wins over the computed delay, because the server + knows better than we do -- but it is still clamped, so a header of 3600 + cannot silently hang the CLI for an hour. + + :param attempt: Which attempt just failed, counting from 1. + :type attempt: int + :param response: The failed response, when there was one. + :type response: Optional[Any] + :param reason: Short description of what failed, for the notice. + :type reason: str + """ + delay = min(_BACKOFF_BASE * (2 ** (attempt - 1)), _BACKOFF_MAX) + delay += random.uniform(0, delay / 2) + + if response is not None: + headers = getattr(response, 'headers', None) or {} + header = headers.get('Retry-After') + if header: + try: + delay = min(float(header), _BACKOFF_MAX) + except (TypeError, ValueError): + pass # A date-format Retry-After; the computed backoff stands. + + # stderr, so a retry notice can never contaminate JSON on stdout. + print(f'{reason}; retrying in {delay:.1f}s', file=sys.stderr) + time.sleep(delay) + def get(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any: """ Perform a GET and return the decoded body. diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index 67715a7..36100fd 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -1,5 +1,7 @@ """``sp run`` — list, inspect, and triage CI runs.""" +import base64 +import sys from typing import Any, Dict, List, Optional, Tuple import click @@ -292,14 +294,22 @@ def run_cancel(ctx: click.Context, run_id: int, reason: Optional[str]) -> None: @click.option('--side', type=click.Choice(('expected', 'actual')), default='actual', show_default=True, help='Which side of the comparison to fetch.') @click.option('--format', 'fmt', default=None, help='Response format accepted by the API.') +@click.option('--decode', is_flag=True, default=False, + help='Write the decoded file to stdout instead of the JSON envelope.') @click.pass_context def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: Optional[int], - output_id: Optional[int], side: str, fmt: Optional[str]) -> None: + output_id: Optional[int], side: str, fmt: Optional[str], decode: bool) -> None: """Fetch one side of a result's output file. Resolves the (media sample, regression, output) ids the same way `sp run diff` does, so the hidden ids the web UI needs are not required here. + The API returns the file base64-encoded inside a JSON envelope. --decode + writes the decoded bytes straight to stdout instead, so the subtitle file + can be read or redirected: + + sp run output 9388 11 --decode > actual.srt + Note: for an output that matched, the API answers ``actual`` with a 303 redirect to ``expected`` -- requests follows it, so the expected content is what comes back. @@ -315,12 +325,48 @@ def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: O f'/runs/{run_id}/samples/{media_sample_id}' f'/regression-tests/{reg_id}/outputs/{out_id}/{side}', params=clean_params({'format': fmt})) + if decode: + _write_decoded(payload) + return except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) render(payload, output) +def _write_decoded(payload: Any) -> None: + """ + Write an output envelope's file content to stdout as raw bytes. + + Written to the binary buffer rather than echoed, so the file survives + byte-for-byte: these are subtitle files that may carry CRLF line endings + and a non-UTF-8 encoding, and re-encoding them would corrupt a diff. + + :param payload: The decoded output envelope from the API. + :type payload: Any + :raises ApiError: when the envelope carries no inline content. + """ + content = payload.get('content') if isinstance(payload, dict) else None + if content is None: + raise ApiError( + 'no_content', + 'This output has no inline content to decode; ' + 'fetch it from download_url instead.', 404, + {'download_url': payload.get('download_url') if isinstance(payload, dict) else None}) + + if (payload.get('encoding') or '').lower() == 'base64': + data = base64.b64decode(content) + else: + data = str(content).encode('utf-8') + + stream = getattr(sys.stdout, 'buffer', None) + if stream is None: # a text-only stream, e.g. under a test runner + sys.stdout.write(data.decode('utf-8', errors='replace')) + else: + stream.write(data) + stream.flush() + + @run.command('artifacts') @click.argument('run_id', type=int) @click.option('--type', 'artifact_type', type=click.Choice(ARTIFACT_TYPES), default=None, diff --git a/sp_cli/main.py b/sp_cli/main.py index e5f6c32..7fec0f8 100644 --- a/sp_cli/main.py +++ b/sp_cli/main.py @@ -27,12 +27,14 @@ @click.option('--output', '-o', type=click.Choice(['json', 'table']), default='json', show_default=True, help='Output format.') @click.option('--timeout', type=int, default=30, show_default=True, help='Per-request timeout (seconds).') +@click.option('--retries', type=int, default=2, show_default=True, + help='Extra attempts for a failed GET (timeouts, 429, 5xx). 0 disables retrying.') @click.option('--no-color', is_flag=True, default=False, help='Never colorize table output. Also honours the NO_COLOR environment variable.') @click.version_option(__version__, prog_name='sp') @click.pass_context def cli(ctx: click.Context, base_url: str, token: Optional[str], output: str, - timeout: int, no_color: bool) -> None: + timeout: int, retries: int, no_color: bool) -> None: """AI-friendly CLI for the CCExtractor CI / Sample Platform. Emits JSON by default so it can be driven by agents and scripts. Point it at @@ -48,7 +50,7 @@ def cli(ctx: click.Context, base_url: str, token: Optional[str], output: str, 'run chmod 600 on it.', err=True) ctx.obj = { - 'client': ApiClient(base_url, token=token, timeout=timeout), + 'client': ApiClient(base_url, token=token, timeout=timeout, retries=retries), 'output': output, 'base_url': base_url, 'color': output == 'table' and not no_color, diff --git a/tests/test_cli.py b/tests/test_cli.py index 259f20f..f0e2211 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -940,3 +940,70 @@ def test_asking_for_every_scope_at_once_is_allowed(self, mock_request): self.assertEqual(result.exit_code, 0) self.assertEqual(len(mock_request.call_args.kwargs['json_body']['scopes']), len(TOKEN_SCOPES)) + + +class OutputDecodeTests(unittest.TestCase): + """`run output --decode` writes the file itself, not the JSON envelope. + + The API returns subtitle files base64-encoded inside JSON. Without --decode + the terminal gets a multi-kilobyte blob that no one can read and no diff + tool can consume. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + #: What /runs/{id}/samples/{id} answers, so the ids can be resolved. + DETAIL = {'sample_id': 11, 'regression_test_id': 11, + 'outputs': [{'output_id': 11, 'status': 'fail'}]} + + @mock.patch('sp_cli.client.ApiClient.get') + def test_decode_writes_the_raw_file(self, mock_get): + """The base64 payload comes back out as the original bytes.""" + import base64 + body = '1\r\n00:00:01,000 --> 00:00:02,000\r\nHELLO\r\n' + mock_get.side_effect = [ + self.DETAIL, + {'content': base64.b64encode(body.encode()).decode(), 'encoding': 'base64'}, + ] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11', '--decode']) + + self.assertEqual(result.exit_code, 0) + # Asserted on bytes: CRLF is what a subtitle file actually contains, and + # result.stdout would normalise it away and hide a re-encoding bug. + self.assertEqual(result.stdout_bytes, body.encode()) + # No JSON envelope leaked alongside the file. + self.assertNotIn(b'"encoding"', result.stdout_bytes) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_plain_content_is_passed_through(self, mock_get): + """An envelope that is not base64 is written as-is.""" + mock_get.side_effect = [self.DETAIL, {'content': 'already text', 'encoding': None}] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11', '--decode']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.stdout, 'already text') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_without_the_flag_the_envelope_is_unchanged(self, mock_get): + """The default stays JSON, so existing scripts are unaffected.""" + mock_get.side_effect = [self.DETAIL, {'content': 'eA==', 'encoding': 'base64'}] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(json.loads(result.stdout)['encoding'], 'base64') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_an_envelope_with_no_inline_content_is_an_error(self, mock_get): + """A download-only artifact has nothing to decode; say so, don't crash.""" + mock_get.side_effect = [ + self.DETAIL, + {'content': None, 'download_url': 'https://storage.example/x'}, + ] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11', '--decode']) + + self.assertEqual(result.exit_code, 4) + envelope = json.loads(result.stderr) + self.assertEqual(envelope['error']['code'], 'no_content') + self.assertIn('download_url', envelope['error']['details']) diff --git a/tests/test_client.py b/tests/test_client.py index 5eeeb5c..90acb7f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -48,7 +48,9 @@ def test_204_returns_none(self, mock_request): def test_error_codes_map_to_exit_codes(self, mock_request): """Each HTTP error maps to its documented exit code.""" cases = {404: 4, 422: 5, 400: 5, 401: 6, 403: 6, 429: 7} - client = ApiClient('https://host') + # retries=0: this asserts the mapping, not the backoff, and 429 is + # retryable -- leaving the default on would make it sleep for seconds. + client = ApiClient('https://host', retries=0) for status, expected_exit in cases.items(): mock_request.return_value = FakeResponse(status, {'code': 'x', 'message': 'm'}) with self.assertRaises(ApiError) as caught: @@ -67,7 +69,7 @@ def test_token_is_sent_as_bearer_header(self, mock_request): def test_connection_failure(self, mock_request): """A transport failure maps to a connection_error with exit code 3.""" with self.assertRaises(ApiError) as caught: - ApiClient('https://host').get('/runs') + ApiClient('https://host', retries=0).get('/runs') self.assertEqual(caught.exception.code, 'connection_error') self.assertEqual(caught.exception.exit_code, 3) @@ -88,3 +90,121 @@ def test_get_paginated_follows_offset(self, mock_request): ] items = ApiClient('https://host').get_paginated('/runs/9/samples') self.assertEqual(items, [1, 2, 3, 4, 5]) + + +class RetryTests(unittest.TestCase): + """Retry behaviour for transient failures. + + Motivated by a live run: `investigate --with-history` makes one call per + failing sample, and a single 30s read timeout partway through threw away + every lookup before it. + """ + + def setUp(self): + """Never actually sleep in tests.""" + patcher = mock.patch('sp_cli.client.time.sleep') + self.sleep = patcher.start() + self.addCleanup(patcher.stop) + + @mock.patch('requests.Session.request') + def test_a_timeout_is_retried_and_can_succeed(self, mock_request): + """One blip must not lose the whole call.""" + mock_request.side_effect = [ + requests.exceptions.ReadTimeout('timed out'), + FakeResponse(200, {'run_id': 9}), + ] + result = ApiClient('https://host').get('/runs/9') + + self.assertEqual(result, {'run_id': 9}) + self.assertEqual(mock_request.call_count, 2) + self.sleep.assert_called_once() + + @mock.patch('requests.Session.request') + def test_retries_are_bounded_and_then_raise(self, mock_request): + """After the budget is spent the original error still surfaces.""" + mock_request.side_effect = requests.exceptions.ReadTimeout('timed out') + with self.assertRaises(ApiError) as caught: + ApiClient('https://host', retries=2).get('/runs/9') + + self.assertEqual(caught.exception.exit_code, 3) + self.assertEqual(mock_request.call_count, 3) # 1 attempt + 2 retries + + @mock.patch('requests.Session.request') + def test_writes_are_never_retried(self, mock_request): + """POST /runs is not idempotent; a retry could queue the run twice.""" + mock_request.side_effect = requests.exceptions.ReadTimeout('timed out') + with self.assertRaises(ApiError): + ApiClient('https://host', retries=5).request('POST', '/runs', json_body={}) + + self.assertEqual(mock_request.call_count, 1) + + @mock.patch('requests.Session.request') + def test_deletes_are_never_retried(self, mock_request): + """A repeated DELETE turns a success into a confusing 404.""" + mock_request.side_effect = requests.exceptions.ReadTimeout('timed out') + with self.assertRaises(ApiError): + ApiClient('https://host', retries=5).delete('/regression-tests/18') + + self.assertEqual(mock_request.call_count, 1) + + @mock.patch('requests.Session.request') + def test_client_errors_are_not_retried(self, mock_request): + """A 404 is deterministic -- asking again just wastes a round trip.""" + mock_request.return_value = FakeResponse(404, {'code': 'not_found', 'message': 'no'}) + with self.assertRaises(ApiError): + ApiClient('https://host', retries=3).get('/runs/9') + + self.assertEqual(mock_request.call_count, 1) + + @mock.patch('requests.Session.request') + def test_server_errors_and_rate_limits_are_retried(self, mock_request): + """502/503/504/429 mean 'not now', so they are worth asking again.""" + for status in (429, 502, 503, 504): + mock_request.reset_mock() + mock_request.side_effect = [ + FakeResponse(status, {'code': 'x', 'message': 'm'}), + FakeResponse(200, {'ok': True}), + ] + result = ApiClient('https://host').get('/runs') + self.assertEqual(result, {'ok': True}, f'status {status}') + self.assertEqual(mock_request.call_count, 2, f'status {status}') + + @mock.patch('requests.Session.request') + def test_retry_after_header_wins_over_the_computed_backoff(self, mock_request): + """The server knows better -- but the value is still clamped.""" + response = FakeResponse(429, {'code': 'x', 'message': 'm'}) + response.headers = {'Retry-After': '7'} + mock_request.side_effect = [response, FakeResponse(200, {})] + ApiClient('https://host').get('/runs') + + self.assertEqual(self.sleep.call_args.args[0], 7.0) + + @mock.patch('requests.Session.request') + def test_an_absurd_retry_after_is_clamped(self, mock_request): + """A header of 3600 must not hang the CLI for an hour.""" + response = FakeResponse(503, {'code': 'x', 'message': 'm'}) + response.headers = {'Retry-After': '3600'} + mock_request.side_effect = [response, FakeResponse(200, {})] + ApiClient('https://host').get('/runs') + + self.assertLessEqual(self.sleep.call_args.args[0], 30.0) + + @mock.patch('requests.Session.request') + def test_a_date_format_retry_after_falls_back_to_backoff(self, mock_request): + """Retry-After may be an HTTP date; that must not crash the retry.""" + response = FakeResponse(503, {'code': 'x', 'message': 'm'}) + response.headers = {'Retry-After': 'Wed, 21 Oct 2026 07:28:00 GMT'} + mock_request.side_effect = [response, FakeResponse(200, {})] + ApiClient('https://host').get('/runs') + + self.assertGreater(self.sleep.call_args.args[0], 0) + + @mock.patch('requests.Session.request') + def test_retries_can_be_disabled(self, mock_request): + """--retries 0 restores the old fail-fast behaviour.""" + mock_request.side_effect = requests.exceptions.ReadTimeout('timed out') + with self.assertRaises(ApiError): + ApiClient('https://host', retries=0).get('/runs') + + self.assertEqual(mock_request.call_count, 1) + self.sleep.assert_not_called() From 7729f336c835b9671ee79541d838415f1144de61 Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Thu, 6 Aug 2026 00:34:14 +0530 Subject: [PATCH 09/10] fix: degrade gracefully when a sample's history is unavailable Verified against production: /samples/{id}/history returns 504 there, and one failed lookup aborted the entire investigation with exit 3 -- discarding the run summary and all 45 classified failures to report a single missing verdict. The classification is the bulk of the answer and it was already in hand. A failed lookup now marks that row UNKNOWN and carries on, and the report gains a `history_incomplete` block naming the samples that could not be fetched, so the gap is visible rather than silently absent. After three consecutive failures the endpoint is treated as down and the remaining rows are marked without calling it. Without that circuit breaker a broken endpoint costs one full timeout per failing sample -- 45 samples at 30s is a 22-minute hang for an answer that is not coming. Run 9388 on production now completes in ~1.5 minutes with all 45 failures classified (24 MISSING_OUTPUT, 20 OUTPUT_DIFF, 1 SEGFAULT), where before it returned nothing at all. Note this is damage control, not a fix for the underlying problem: the verdicts are all UNKNOWN on production because the endpoint paginates in Python after loading every result for the sample across every run, so ?limit=5 does not reduce the work. That needs fixing in sample-platform. --- .claude/settings.local.json | 9 +++++ sp_cli/commands/investigate.py | 67 ++++++++++++++++++++++++++----- tests/test_cli.py | 73 ++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 .claude/settings.local.json diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..8632644 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(python3 *)", + "Read(//Users/gauravkarmakar/sp-cc/sample-platform/**)", + "Bash(sed -E 's/=.*/= /' config.py)" + ] + } +} diff --git a/sp_cli/commands/investigate.py b/sp_cli/commands/investigate.py index b2c7f3c..d8df9c6 100644 --- a/sp_cli/commands/investigate.py +++ b/sp_cli/commands/investigate.py @@ -14,6 +14,11 @@ _RUN_FIELDS = ('run_id', 'pr_number', 'platform', 'commit_sha', 'branch', 'status', 'github_link') +#: Consecutive failed history lookups before the endpoint is treated as down. +#: Without this a broken endpoint costs one full timeout per failing sample -- +#: 45 samples x 30s is a 22-minute hang for an answer that will not come. +_HISTORY_FAILURE_LIMIT = 3 + #: How many prior runs to weigh per failure when --with-history is used. Deep #: enough to see a sample settle, shallow enough to keep it one call per sample. DEFAULT_HISTORY_DEPTH = 20 @@ -63,13 +68,11 @@ def investigate(ctx: click.Context, run_id: int, with_history: bool, } if with_history: - try: - with Spinner('Fetching sample history', output != 'json'): - _attach_history(client, failures, run_id, run.get('platform'), depth) - except ApiError as error: - render_error(error, output) - raise SystemExit(error.exit_code) + with Spinner('Fetching sample history', output != 'json'): + degraded = _attach_history(client, failures, run_id, run.get('platform'), depth) report['by_verdict'] = group_by_verdict(failures) + if degraded: + report['history_incomplete'] = degraded if output == 'json': render(report, 'json') @@ -78,7 +81,7 @@ def investigate(ctx: click.Context, run_id: int, with_history: bool, def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, - platform: Optional[str], depth: int) -> None: + platform: Optional[str], depth: int) -> Optional[Dict[str, Any]]: """ Add a ``history`` verdict block to every failure row, in place. @@ -87,6 +90,13 @@ def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, by regression test id. Restricting to the run's own platform keeps a Windows failure from being judged against Linux history. + A lookup that fails degrades that one row to UNKNOWN rather than aborting: + the classified failures and the run summary are the bulk of the answer, and + throwing all of it away because one sample's history is unavailable is worse + than reporting the gap. After ``_HISTORY_FAILURE_LIMIT`` consecutive + failures the endpoint is treated as down and the rest are marked without + calling it -- otherwise a broken endpoint costs one full timeout per sample. + :param client: The API client. :type client: Any :param failures: Classified failure rows, mutated in place. @@ -97,24 +107,54 @@ def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, :type platform: Optional[str] :param depth: How many prior runs to consider per failure. :type depth: int + :return: A summary of what could not be fetched, or ``None`` if all of it was. + :rtype: Optional[Dict[str, Any]] """ cache: Dict[int, List[Dict[str, Any]]] = {} + consecutive_failures = 0 + given_up = False + failed_samples: List[int] = [] + last_error: Optional[str] = None + for failure in failures: sample_id = failure.get('sample_id') if not isinstance(sample_id, int): failure['history'] = unknown_history('Result has no sample id to look up') continue + if given_up: + failure['history'] = unknown_history( + 'Skipped: the history endpoint is not responding') + continue + if sample_id not in cache: # +1 so the current run's own entry cannot displace an older one. params = clean_params({'platform': platform, 'limit': depth + 1}) - cache[sample_id] = client.get_paginated( - f'/samples/{sample_id}/history', params=params, max_items=depth + 1) + try: + cache[sample_id] = client.get_paginated( + f'/samples/{sample_id}/history', params=params, max_items=depth + 1) + consecutive_failures = 0 + except ApiError as error: + consecutive_failures += 1 + last_error = error.message + failed_samples.append(sample_id) + failure['history'] = unknown_history(f'History lookup failed: {error.code}') + if consecutive_failures >= _HISTORY_FAILURE_LIMIT: + given_up = True + continue current, prior = split_history(cache[sample_id], run_id, failure.get('regression_test_id')) failure['history'] = classify_history(current, prior[:depth]) + if not failed_samples: + return None + return { + 'failed_samples': failed_samples, + 'gave_up': given_up, + 'reason': last_error, + } + def _print_digest(report: Dict[str, Any], with_history: bool = False, color: bool = False) -> None: @@ -150,6 +190,15 @@ def _print_digest(report: Dict[str, Any], with_history: bool = False, for verdict, count in by_verdict.items(): click.echo(f" {str(count).rjust(4)} {verdict}") + incomplete = report.get('history_incomplete') + if incomplete: + # To stderr: the digest itself stays the answer, this is a caveat on it. + note = (f" note: history unavailable for {len(incomplete['failed_samples'])} " + f"sample(s); those rows are UNKNOWN") + if incomplete.get('gave_up'): + note += ' (stopped asking after repeated failures)' + click.echo(note, err=True) + failures: List[Dict[str, Any]] = report['failures'] if failures: click.echo() diff --git a/tests/test_cli.py b/tests/test_cli.py index f0e2211..6ceef4a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1007,3 +1007,76 @@ def test_an_envelope_with_no_inline_content_is_an_error(self, mock_get): envelope = json.loads(result.stderr) self.assertEqual(envelope['error']['code'], 'no_content') self.assertIn('download_url', envelope['error']['details']) + + +class HistoryDegradationTests(unittest.TestCase): + """A failed history lookup must not discard the whole investigation. + + Found against production: /samples/{id}/history 504s there, and one failure + aborted the entire command -- throwing away the run summary and all 45 + classified failures to report a single missing verdict. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + RUN = {'run_id': 9388, 'platform': 'linux', 'status': 'fail'} + SUMMARY = {'fail_count': 2, 'total_samples': 2, 'pass_count': 0} + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_one_failed_lookup_does_not_lose_the_investigation(self, mock_get, mock_paginated): + """The classified failures still come back; only that row is UNKNOWN.""" + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [ + SAMPLES_WITH_IDS, + ApiError('connection_error', 'Read timed out.'), + [], + ] + result = self.runner.invoke(cli, ['investigate', '9388', '--with-history']) + + self.assertEqual(result.exit_code, 0, result.output) + report = json.loads(result.stdout) + self.assertEqual(len(report['failures']), 2) + self.assertEqual(report['failures'][0]['history']['verdict'], 'UNKNOWN') + self.assertIn(42, report['history_incomplete']['failed_samples']) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_it_stops_asking_after_repeated_failures(self, mock_get, mock_paginated): + """A down endpoint must not cost one full timeout per sample.""" + many = [dict(s, sample_id=100 + i, regression_test_id=200 + i) + for i, s in enumerate(SAMPLES_WITH_IDS * 5)] + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [many] + [ + ApiError('connection_error', 'Read timed out.') for _ in range(20)] + + result = self.runner.invoke(cli, ['investigate', '9388', '--with-history']) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.stdout) + self.assertTrue(report['history_incomplete']['gave_up']) + # 1 call for the samples list + at most the failure limit before giving up. + self.assertLessEqual(mock_paginated.call_count, 4) + self.assertEqual(len(report['failures']), len(many)) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_clean_run_reports_no_gap(self, mock_get, mock_paginated): + """history_incomplete is absent when every lookup succeeded.""" + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [SAMPLES_WITH_IDS, [], []] + result = self.runner.invoke(cli, ['investigate', '9388', '--with-history']) + + self.assertEqual(result.exit_code, 0) + self.assertNotIn('history_incomplete', json.loads(result.stdout)) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_the_run_itself_failing_is_still_fatal(self, mock_get, mock_paginated): + """Degrading history is fine; a missing run means there is no answer at all.""" + mock_get.side_effect = ApiError('not_found', 'Run 1 not found', 404) + result = self.runner.invoke(cli, ['investigate', '1', '--with-history']) + + self.assertEqual(result.exit_code, 4) From 843365d96eea4489250a3feeddbc5829835df4b1 Mon Sep 17 00:00:00 2001 From: GAURAV KARMAKAR Date: Thu, 6 Aug 2026 01:56:09 +0530 Subject: [PATCH 10/10] fix: ten defects found by code review A high-effort review of the branch confirmed ten defects. They cluster in three areas, and several were in code written an hour earlier -- completing a production run was not evidence it was correct. Credential handling Running `pytest` overwrote the developer's real ~/.config/sp/config.json: one login test omitted --no-save and CliCommandTests has no XDG isolation. It had already fired on this machine, leaving the literal `spci_x` behind. Adding --no-save to that one test is not the fix -- relying on every future test remembering it is what failed. tests/conftest.py now redirects XDG_CONFIG_HOME and HOME for every test automatically. `auth logout` cleared the saved session unconditionally, so revoking a scratch token from SP_API_TOKEN deleted an unrelated 30-day credential that cannot be recovered. It now compares the effective token against the saved one by value. The error path splits too: 401/403 still clears, because the token is dead either way, but a dropped connection leaves it alone rather than discarding a token that is probably fine. save_token relied on os.open's mode, which is ignored for a file that already exists -- so re-login wrote a fresh token into a still-0644 file and chmod-ed it afterwards. It now fchmods the descriptor before any byte is written. investigate --with-history Three bugs that only appear with more than one failing regression test per sample, which is why the original tests missed them: a failed lookup was never memoised, so it was re-called and re-counted for every row sharing that sample and one bad sample tripped the breaker; the given_up check preceded the cache check, discarding history already fetched successfully; and --history-depth was unbounded although the API rejects limit > 100, so --history-depth 100 silently disabled all history while still exiting 0. The window itself was also wrong, and only half of that is fixable here. The endpoint pages over *all* of a sample's regression tests and slices before the CLI can filter, so a page of N gave roughly N/(tests on the sample) runs of the test under investigation -- reporting NEVER_PASSED for a test that passed nine runs ago. The CLI now requests a full page and applies the depth after filtering, and when a saturated page still leaves a short window it sets window_truncated and drops NEVER_PASSED to low confidence: "never passed" and "did not pass in the few runs I could see" are different claims. A real fix needs sample-platform#1161, and the docs now say so. Output contracts --decode ignored the envelope's truncated flag, writing a file that ends mid-stream and looks complete -- every diff against it reports spurious missing lines. It is now refused, naming download_url, with --allow-truncated as an explicit opt-in. `auth revoke` and `auth logout` printed English sentences to stdout, so piping either into jq failed while every other command worked. Both now render JSON. LOG_MAX_LIMIT mirrored a 1-500 clamp that sits behind a validator rejecting anything over 100, so it was unreachable and the help advertised a page size the API refuses. Deleted rather than corrected: it duplicated a rule that is uniform across both paginators. That audit found 33 other unvalidated --limit/--offset options, now all bounded by IntRange. The subtle fixes were revert-checked rather than trusted: restoring the old code makes the new tests fail, including "token written while mode was 0o644". 182 tests, up from 166. --- README.md | 8 + sp_cli/commands/admin.py | 15 +- sp_cli/commands/auth.py | 47 ++++-- sp_cli/commands/category.py | 9 +- sp_cli/commands/investigate.py | 56 +++++-- sp_cli/commands/regression.py | 8 +- sp_cli/commands/run.py | 83 +++++++--- sp_cli/commands/sample.py | 16 +- sp_cli/commands/system.py | 9 +- sp_cli/config.py | 44 +++-- sp_cli/constants.py | 12 +- sp_cli/history.py | 24 ++- tests/conftest.py | 36 ++++ tests/test_cli.py | 292 ++++++++++++++++++++++++++++++++- tests/test_history.py | 55 +++++++ tests/test_ux.py | 149 +++++++++++++++++ 16 files changed, 768 insertions(+), 95 deletions(-) create mode 100644 tests/conftest.py diff --git a/README.md b/README.md index ef68e4a..93c4365 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,14 @@ agent gets a straight answer about *why* a test failed, without reading logs. With `--with-history`, each failure also gets a verdict across previous runs: `NEW_REGRESSION`, `STILL_FAILING`, `NEVER_PASSED`, `FLAKY`, `NO_HISTORY`. +How far back that verdict can see depends on the sample. The history endpoint +pages over every regression test defined on a sample, so a test sharing its +sample with many others gets a shorter effective window than `--history-depth` +asks for. When that happens the verdict carries `window_truncated: true` and +`NEVER_PASSED` is reported at low confidence — it means "did not pass in the +runs visible here", not "has never passed". Check `prior_runs_considered` for +the window a verdict was actually based on. + ### Reliability Failed `GET`s are retried with exponential backoff — connection failures, read diff --git a/sp_cli/commands/admin.py b/sp_cli/commands/admin.py index 3f76e83..6157163 100644 --- a/sp_cli/commands/admin.py +++ b/sp_cli/commands/admin.py @@ -9,7 +9,8 @@ import click from sp_cli.constants import (BLOCKED_USER_COMMENT_MAX_LENGTH, - EXTENSION_MAX_LENGTH, PLATFORMS) + EXTENSION_MAX_LENGTH, MAX_OFFSET, MAX_PAGE_LIMIT, + PLATFORMS) from sp_cli.runner import clean_params, fetch_and_render, send_and_render @@ -56,8 +57,10 @@ def blocked_users() -> None: @blocked_users.command('ls') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def blocked_users_ls(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: """List the GitHub accounts blocked from triggering CI runs.""" @@ -99,8 +102,10 @@ def forbidden_extensions() -> None: @forbidden_extensions.command('ls') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def forbidden_extensions_ls(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: diff --git a/sp_cli/commands/auth.py b/sp_cli/commands/auth.py index 5b5c722..edcb342 100644 --- a/sp_cli/commands/auth.py +++ b/sp_cli/commands/auth.py @@ -6,8 +6,8 @@ from sp_cli import config from sp_cli.client import ApiError -from sp_cli.constants import (TOKEN_MAX_DAYS, TOKEN_MIN_DAYS, TOKEN_SCOPES, - USER_ROLES) +from sp_cli.constants import (MAX_OFFSET, MAX_PAGE_LIMIT, TOKEN_MAX_DAYS, + TOKEN_MIN_DAYS, TOKEN_SCOPES, USER_ROLES) from sp_cli.output import render, render_error from sp_cli.runner import clean_params, fetch_and_render, send_and_render @@ -61,8 +61,10 @@ def auth_login(ctx: click.Context, email: str, password: str, token_name: str, @auth.command('tokens') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def auth_tokens(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: """List this account's API tokens (metadata only, never the token itself).""" @@ -86,7 +88,11 @@ def auth_revoke(ctx: click.Context, token_id: int) -> None: except ApiError as error: render_error(error, output) raise SystemExit(error.exit_code) - click.echo(f'Token {token_id} revoked.') + # The API answers 204 with no body, so the result is synthesized -- but it + # is still rendered, because every other command emits JSON on stdout and a + # bare English sentence breaks anything piping this into jq. The shape + # mirrors the API's own deletes, which answer {id, deleted}. + render({'token_id': token_id, 'revoked': True}, output, ctx.obj.get('color', False)) @auth.command('logout') @@ -94,19 +100,34 @@ def auth_revoke(ctx: click.Context, token_id: int) -> None: def auth_logout(ctx: click.Context) -> None: """Revoke the current API token and drop the saved session. - The local file is cleared even if the server call fails, so a token that - was already revoked or expired cannot leave a stale credential on disk. + The saved file is only cleared when the token being revoked is the saved + one. Revoking a scratch token passed via --token or SP_API_TOKEN leaves an + unrelated saved session alone -- deleting it would be unrecoverable, since + the plaintext token is returned only once at creation. + + If the server rejects the token as dead (401/403) the saved copy is cleared + too, since it cannot work again. A connection failure leaves it in place: + the token may still be perfectly good and only the network was at fault. """ client = ctx.obj['client'] output = ctx.obj['output'] + + # Compared by value rather than tracking provenance, so passing --token + # with the same value as the saved session still clears it correctly. + saved = config.saved_token() + revoking_saved = saved is not None and saved == client.token + try: client.request('DELETE', '/auth/tokens/current') except ApiError as error: - config.clear_token() + if revoking_saved and error.status in (401, 403): + config.clear_token() render_error(error, output) raise SystemExit(error.exit_code) - cleared = config.clear_token() - click.echo('Token revoked.' + (' Saved session cleared.' if cleared else '')) + + cleared = bool(revoking_saved and config.clear_token()) + render({'revoked': True, 'saved_session_cleared': cleared}, + output, ctx.obj.get('color', False)) @auth.command('whoami') @@ -121,8 +142,10 @@ def auth_whoami(ctx: click.Context) -> None: @auth.command('users') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def auth_users(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: """List platform users, oldest first (admin only). diff --git a/sp_cli/commands/category.py b/sp_cli/commands/category.py index 09b58fc..271380a 100644 --- a/sp_cli/commands/category.py +++ b/sp_cli/commands/category.py @@ -4,7 +4,8 @@ import click -from sp_cli.constants import DESCRIPTION_MAX_LENGTH, NAME_MAX_LENGTH +from sp_cli.constants import (DESCRIPTION_MAX_LENGTH, MAX_OFFSET, + MAX_PAGE_LIMIT, NAME_MAX_LENGTH) from sp_cli.runner import clean_params, fetch_and_render, send_and_render @@ -14,8 +15,10 @@ def category() -> None: @category.command('ls') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def category_ls(ctx: click.Context, limit: Optional[int], offset: Optional[int]) -> None: """List categories alphabetically, each with how many tests reference it.""" diff --git a/sp_cli/commands/investigate.py b/sp_cli/commands/investigate.py index d8df9c6..cffa665 100644 --- a/sp_cli/commands/investigate.py +++ b/sp_cli/commands/investigate.py @@ -5,6 +5,7 @@ import click from sp_cli.client import ApiError +from sp_cli.constants import MAX_PAGE_LIMIT from sp_cli.history import (NEW_REGRESSION, classify_history, group_by_verdict, split_history, unknown_history) from sp_cli.output import render, render_error @@ -28,9 +29,9 @@ @click.argument('run_id', type=int) @click.option('--with-history', 'with_history', is_flag=True, default=False, help='Label each failure as a new regression, long-standing, or never-passing.') -@click.option('--history-depth', type=int, default=None, - help=f'Prior runs to weigh per failure (default: {DEFAULT_HISTORY_DEPTH}). ' - 'Implies --with-history.') +@click.option('--history-depth', type=click.IntRange(1, MAX_PAGE_LIMIT - 1), default=None, + help=f'Prior runs to weigh per failure (default: {DEFAULT_HISTORY_DEPTH}, ' + f'max {MAX_PAGE_LIMIT - 1}). Implies --with-history.') @click.pass_context def investigate(ctx: click.Context, run_id: int, with_history: bool, history_depth: Optional[int]) -> None: @@ -111,9 +112,13 @@ def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, :rtype: Optional[Dict[str, Any]] """ cache: Dict[int, List[Dict[str, Any]]] = {} + # Failures are remembered per sample too. Several regression tests share one + # sample, so without this a single unreachable sample is re-fetched once per + # row -- paying the timeout again each time and counting itself repeatedly + # toward the breaker, which then trips on one sample rather than three. + failed: Dict[int, str] = {} consecutive_failures = 0 given_up = False - failed_samples: List[int] = [] last_error: Optional[str] = None for failure in failures: @@ -122,35 +127,54 @@ def _attach_history(client: Any, failures: List[Dict[str, Any]], run_id: int, failure['history'] = unknown_history('Result has no sample id to look up') continue - if given_up: - failure['history'] = unknown_history( - 'Skipped: the history endpoint is not responding') + if sample_id in failed: + failure['history'] = unknown_history(f'History lookup failed: {failed[sample_id]}') continue if sample_id not in cache: - # +1 so the current run's own entry cannot displace an older one. - params = clean_params({'platform': platform, 'limit': depth + 1}) + # Checked here rather than at the top of the loop: giving up must + # only stop *new* requests. A sample already fetched successfully + # can still be classified from memory. + if given_up: + failure['history'] = unknown_history( + 'Skipped: the history endpoint is not responding') + continue + + # A full page, not depth + 1. The endpoint returns entries for + # *every* regression test defined on the sample and slices only + # afterwards, so a page of N covers roughly N/(tests on this sample) + # runs of the one test being investigated -- asking for depth + 1 + # silently delivered a fraction of it. Requesting the maximum costs + # the server nothing today, because it loads the sample's whole + # history and paginates in Python either way (sample-platform#1161). + params = clean_params({'platform': platform, 'limit': MAX_PAGE_LIMIT}) try: cache[sample_id] = client.get_paginated( - f'/samples/{sample_id}/history', params=params, max_items=depth + 1) + f'/samples/{sample_id}/history', params=params, + max_items=MAX_PAGE_LIMIT) consecutive_failures = 0 except ApiError as error: consecutive_failures += 1 last_error = error.message - failed_samples.append(sample_id) + failed[sample_id] = error.code failure['history'] = unknown_history(f'History lookup failed: {error.code}') if consecutive_failures >= _HISTORY_FAILURE_LIMIT: given_up = True continue - current, prior = split_history(cache[sample_id], run_id, - failure.get('regression_test_id')) - failure['history'] = classify_history(current, prior[:depth]) + entries = cache[sample_id] + current, prior = split_history(entries, run_id, failure.get('regression_test_id')) + # A saturated page means the sample's history continues past what was + # read, so a window shorter than requested is a limit of the read rather + # than of the test's actual history. NEVER_PASSED must not be asserted + # confidently on that basis. + truncated = len(entries) >= MAX_PAGE_LIMIT and len(prior) < depth + failure['history'] = classify_history(current, prior[:depth], truncated) - if not failed_samples: + if not failed: return None return { - 'failed_samples': failed_samples, + 'failed_samples': sorted(failed), 'gave_up': given_up, 'reason': last_error, } diff --git a/sp_cli/commands/regression.py b/sp_cli/commands/regression.py index b0a11dc..e3708a5 100644 --- a/sp_cli/commands/regression.py +++ b/sp_cli/commands/regression.py @@ -6,7 +6,7 @@ from sp_cli.constants import (COMMAND_MAX_LENGTH, DESCRIPTION_MAX_LENGTH, EXPECTED_RC_MAX, EXPECTED_RC_MIN, INPUT_TYPES, - OUTPUT_TYPES) + MAX_OFFSET, MAX_PAGE_LIMIT, OUTPUT_TYPES) from sp_cli.runner import clean_params, fetch_and_render, send_and_render @@ -21,8 +21,10 @@ def regression() -> None: @click.option('--active/--inactive', 'active', default=None, help='Select active or inactive tests (default: active only).') @click.option('--sample-id', type=int, default=None, help='Filter by sample id.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def regression_ls(ctx: click.Context, category: Optional[str], tag: Optional[str], active: Optional[bool], sample_id: Optional[int], diff --git a/sp_cli/commands/run.py b/sp_cli/commands/run.py index 36100fd..520369e 100644 --- a/sp_cli/commands/run.py +++ b/sp_cli/commands/run.py @@ -10,8 +10,8 @@ from sp_cli.constants import (ARTIFACT_TYPES, CANCEL_REASON_MIN_LENGTH, COMMIT_SHA_LENGTH, ERROR_GROUP_BY, ERROR_SEVERITIES, ERROR_TYPES, INFRA_ERROR_TYPES, - LOG_CONTAINS_MAX_LENGTH, LOG_LEVELS, - LOG_MAX_LIMIT, LOG_SOURCES, + LOG_CONTAINS_MAX_LENGTH, LOG_LEVELS, LOG_SOURCES, + MAX_OFFSET, MAX_PAGE_LIMIT, MAX_REGRESSION_TEST_IDS, PLATFORMS, RUN_STATUSES, SAMPLE_STATUSES) from sp_cli.output import render, render_error @@ -37,8 +37,10 @@ def run() -> None: help='Only runs first seen at/after this time (ISO 8601).') @click.option('--created-before', 'created_before', default=None, help='Only runs first seen at/before this time (ISO 8601).') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_ls(ctx: click.Context, status: Optional[str], platform: Optional[str], branch: Optional[str], commit_sha: Optional[str], repository: Optional[str], sort: Optional[str], @@ -137,8 +139,10 @@ def run_failures(ctx: click.Context, run_id: int) -> None: @click.option('--name', default=None, help='Substring match on the sample name.') @click.option('--tag', default=None, help='Filter by sample tag.') @click.option('--category', default=None, help='Filter by regression-test category.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_results(ctx: click.Context, run_id: int, status: Optional[str], name: Optional[str], tag: Optional[str], category: Optional[str], @@ -241,8 +245,10 @@ def run_approve_baseline(ctx: click.Context, run_id: int, sample_id: int, @run.command('progress') @click.argument('run_id', type=int) -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_progress(ctx: click.Context, run_id: int, limit: Optional[int], offset: Optional[int]) -> None: @@ -296,9 +302,13 @@ def run_cancel(ctx: click.Context, run_id: int, reason: Optional[str]) -> None: @click.option('--format', 'fmt', default=None, help='Response format accepted by the API.') @click.option('--decode', is_flag=True, default=False, help='Write the decoded file to stdout instead of the JSON envelope.') +@click.option('--allow-truncated', is_flag=True, default=False, + help='With --decode, write the first 1 MiB of an oversized output ' + 'instead of refusing it. The file will be incomplete.') @click.pass_context def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: Optional[int], - output_id: Optional[int], side: str, fmt: Optional[str], decode: bool) -> None: + output_id: Optional[int], side: str, fmt: Optional[str], decode: bool, + allow_truncated: bool) -> None: """Fetch one side of a result's output file. Resolves the (media sample, regression, output) ids the same way `sp run @@ -326,7 +336,7 @@ def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: O f'/regression-tests/{reg_id}/outputs/{out_id}/{side}', params=clean_params({'format': fmt})) if decode: - _write_decoded(payload) + _write_decoded(payload, allow_truncated) return except ApiError as error: render_error(error, output) @@ -334,7 +344,7 @@ def run_output(ctx: click.Context, run_id: int, sample_id: int, regression_id: O render(payload, output) -def _write_decoded(payload: Any) -> None: +def _write_decoded(payload: Any, allow_truncated: bool = False) -> None: """ Write an output envelope's file content to stdout as raw bytes. @@ -342,17 +352,36 @@ def _write_decoded(payload: Any) -> None: byte-for-byte: these are subtitle files that may carry CRLF line endings and a non-UTF-8 encoding, and re-encoding them would corrupt a diff. + The API inlines at most 1 MiB and sets ``truncated`` beyond that. Writing + the fragment anyway would produce a file that ends mid-stream but looks + complete, and every diff run against it would report spurious missing lines + at the end -- so it is refused unless explicitly allowed. The envelope's + ``sha256`` is computed over the whole file, so a truncated write cannot even + be checked against it. + :param payload: The decoded output envelope from the API. :type payload: Any - :raises ApiError: when the envelope carries no inline content. + :param allow_truncated: Write the partial content instead of refusing it. + :type allow_truncated: bool + :raises ApiError: when there is no inline content, or it is incomplete. """ content = payload.get('content') if isinstance(payload, dict) else None + download_url = payload.get('download_url') if isinstance(payload, dict) else None if content is None: raise ApiError( 'no_content', 'This output has no inline content to decode; ' 'fetch it from download_url instead.', 404, - {'download_url': payload.get('download_url') if isinstance(payload, dict) else None}) + {'download_url': download_url}) + + if payload.get('truncated') and not allow_truncated: + raise ApiError( + 'output_truncated', + 'This output exceeds the API\'s 1 MiB inline limit, so decoding it ' + 'would write an incomplete file. Download the whole file from ' + 'download_url, or pass --allow-truncated to write the first 1 MiB.', + None, + {'download_url': download_url, 'sha256': payload.get('sha256')}) if (payload.get('encoding') or '').lower() == 'base64': data = base64.b64decode(content) @@ -371,8 +400,10 @@ def _write_decoded(payload: Any) -> None: @click.argument('run_id', type=int) @click.option('--type', 'artifact_type', type=click.Choice(ARTIFACT_TYPES), default=None, help='Filter by artifact type.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_artifacts(ctx: click.Context, run_id: int, artifact_type: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: @@ -395,8 +426,8 @@ def run_artifacts(ctx: click.Context, run_id: int, artifact_type: Optional[str], help='Keep lines from this component.') @click.option('--contains', default=None, help=f'Case-insensitive substring filter (max {LOG_CONTAINS_MAX_LENGTH} chars).') -@click.option('--limit', type=int, default=None, - help=f'Lines per page (max {LOG_MAX_LIMIT}, default 100).') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Lines per page (1-{MAX_PAGE_LIMIT}, default 100).') @click.option('--cursor', default=None, help='Resume from a previous response\'s next_cursor.') @click.option('--all', 'fetch_all', is_flag=True, default=False, help='Follow next_cursor and return the whole log at once.') @@ -449,8 +480,10 @@ def run_logs(ctx: click.Context, run_id: int, level: Optional[str], source: Opti help='Filter by severity. Test errors are only error or warning.') @click.option('--sample', 'sample_id', type=int, default=None, help='Restrict to one media sample id.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_errors(ctx: click.Context, run_id: int, error_type: Optional[str], severity: Optional[str], sample_id: Optional[int], @@ -472,8 +505,10 @@ def run_errors(ctx: click.Context, run_id: int, error_type: Optional[str], help='Bucket key. The API defaults to type.') @click.option('--severity', type=click.Choice(ERROR_SEVERITIES), default=None, help='Keep only buckets at this severity.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_error_summary(ctx: click.Context, run_id: int, group_by: Optional[str], severity: Optional[str], limit: Optional[int], @@ -497,8 +532,10 @@ def run_error_summary(ctx: click.Context, run_id: int, group_by: Optional[str], help='Filter by severity. These are always reported as critical.') @click.option('--include-stack', is_flag=True, default=False, help='Include stack traces (admin or contributor only; 403 otherwise).') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def run_infra_errors(ctx: click.Context, run_id: int, error_type: Optional[str], severity: Optional[str], include_stack: bool, diff --git a/sp_cli/commands/sample.py b/sp_cli/commands/sample.py index 1dfb592..d7b6e0d 100644 --- a/sp_cli/commands/sample.py +++ b/sp_cli/commands/sample.py @@ -4,8 +4,8 @@ import click -from sp_cli.constants import (PLATFORMS, SAMPLE_CATALOG_STATUSES, - SAMPLE_STATUSES) +from sp_cli.constants import (MAX_OFFSET, MAX_PAGE_LIMIT, PLATFORMS, + SAMPLE_CATALOG_STATUSES, SAMPLE_STATUSES) from sp_cli.runner import clean_params, fetch_and_render @@ -21,8 +21,10 @@ def sample() -> None: @click.option('--sha256', default=None, help='Filter by exact SHA-256 hash.') @click.option('--status', type=click.Choice(SAMPLE_CATALOG_STATUSES), default=None, help='Catalog visibility, not a test outcome.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def sample_ls(ctx: click.Context, name: Optional[str], tag: Optional[str], extension: Optional[str], sha256: Optional[str], status: Optional[str], @@ -64,8 +66,10 @@ def sample_details(ctx: click.Context, sample_id: int) -> None: help='Only runs first seen at/after this time (ISO 8601).') @click.option('--created-before', 'created_before', default=None, help='Only runs first seen at/before this time (ISO 8601).') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def sample_history(ctx: click.Context, sample_id: int, platform: Optional[str], branch: Optional[str], status: Optional[str], created_after: Optional[str], diff --git a/sp_cli/commands/system.py b/sp_cli/commands/system.py index fcca38a..2f87af1 100644 --- a/sp_cli/commands/system.py +++ b/sp_cli/commands/system.py @@ -4,7 +4,8 @@ import click -from sp_cli.constants import PLATFORMS, QUEUE_STATUSES +from sp_cli.constants import (MAX_OFFSET, MAX_PAGE_LIMIT, PLATFORMS, + QUEUE_STATUSES) from sp_cli.runner import clean_params, fetch_and_render @@ -19,8 +20,10 @@ def health(ctx: click.Context) -> None: @click.option('--platform', type=click.Choice(PLATFORMS), default=None, help='Test platform.') @click.option('--status', type=click.Choice(QUEUE_STATUSES), default=None, help='Restrict to one side of the queue.') -@click.option('--limit', type=int, default=None, help='Page size (max 100).') -@click.option('--offset', type=int, default=None, help='Pagination offset.') +@click.option('--limit', type=click.IntRange(1, MAX_PAGE_LIMIT), default=None, + help=f'Page size (1-{MAX_PAGE_LIMIT}).') +@click.option('--offset', type=click.IntRange(0, MAX_OFFSET), default=None, + help='Pagination offset.') @click.pass_context def queue(ctx: click.Context, platform: Optional[str], status: Optional[str], limit: Optional[int], offset: Optional[int]) -> None: diff --git a/sp_cli/config.py b/sp_cli/config.py index 9b51daa..1c8307d 100644 --- a/sp_cli/config.py +++ b/sp_cli/config.py @@ -61,12 +61,39 @@ def saved_token() -> Optional[str]: return token if isinstance(token, str) and token else None +def _write_private(path: Path, data: Dict[str, Any]) -> None: + """ + Write JSON to ``path`` with owner-only permissions in force before any bytes land. + + ``os.open``'s mode argument only applies when the file is *created*; an + existing file keeps whatever permissions it already had. Writing first and + ``chmod``-ing afterwards therefore leaves a window in which a fresh token + sits in a still-world-readable file. Tightening the open descriptor with + ``fchmod`` before writing closes that window, and works on the descriptor so + there is no path-swap race either. + + :param path: The file to write. + :type path: Path + :param data: The mapping to serialize. + :type data: Dict[str, Any] + """ + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _FILE_MODE) + with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: + if hasattr(os, 'fchmod'): + os.fchmod(handle.fileno(), _FILE_MODE) + json.dump(data, handle, indent=2) + handle.write('\n') + if not hasattr(os, 'fchmod'): # Windows: no descriptor-level chmod available. + os.chmod(path, _FILE_MODE) + + def save_token(token: str, base_url: Optional[str] = None) -> Path: """ Persist a token (and optionally the base URL it belongs to) at mode 0600. - The file is created with restrictive permissions from the outset rather - than chmod-ed afterwards, so the secret is never briefly world-readable. + The permissions are tightened before the token is written, so the secret is + never briefly world-readable -- including on re-login, when the file already + exists and ``os.open``'s creation mode would be ignored. :param token: The plaintext bearer token to store. :type token: str @@ -83,13 +110,7 @@ def save_token(token: str, base_url: Optional[str] = None) -> Path: if base_url: data['base_url'] = base_url - # Open through os.open so the mode applies at creation time. An existing - # file keeps its inode, so re-chmod it too. - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _FILE_MODE) - with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: - json.dump(data, handle, indent=2) - handle.write('\n') - os.chmod(path, _FILE_MODE) + _write_private(path, data) return path @@ -113,10 +134,7 @@ def clear_token() -> bool: return False return True - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _FILE_MODE) - with os.fdopen(descriptor, 'w', encoding='utf-8') as handle: - json.dump(data, handle, indent=2) - handle.write('\n') + _write_private(path, data) return True diff --git a/sp_cli/constants.py b/sp_cli/constants.py index 72f7646..c35e26e 100644 --- a/sp_cli/constants.py +++ b/sp_cli/constants.py @@ -40,6 +40,15 @@ #: ``POST /runs/{id}/cancel`` rejects a reason shorter than this. CANCEL_REASON_MIN_LENGTH = 5 +#: ``mod_api.middleware.validation._parse_limit`` 400s on ``limit < 1 or +#: limit > 100``, for both the offset and cursor paginators. This is the real +#: ceiling on every ``--limit``; a service-level clamp behind it is unreachable. +MAX_PAGE_LIMIT = 100 + +#: ``validate_offset_pagination`` rejects a negative offset and anything above +#: this (a signed 32-bit maximum). +MAX_OFFSET = 2147483647 + #: ``GET /runs/{id}/errors`` — the types ``derive_errors_for_run`` can emit. #: Test errors are derived from result rows, not stored, so this is the closed set. ERROR_TYPES = ('exit_code_mismatch', 'missing_output', 'diff_mismatch') @@ -65,9 +74,6 @@ #: ``GET /runs/{id}/logs`` — ``_extract_source`` keywords; unmatched lines are ``web``. LOG_SOURCES = ('orchestrator', 'worker', 'build', 'test_runner', 'web') -#: ``read_log_lines`` clamps the page size into this range server-side. -LOG_MAX_LIMIT = 500 - #: ``GET /runs/{id}/logs`` rejects a longer ``contains`` filter with a 400. LOG_CONTAINS_MAX_LENGTH = 100 diff --git a/sp_cli/history.py b/sp_cli/history.py index 4be0008..f0726a4 100644 --- a/sp_cli/history.py +++ b/sp_cli/history.py @@ -89,14 +89,23 @@ def _count_transitions(chronological: List[Dict[str, Any]]) -> int: def classify_history(current: Optional[Dict[str, Any]], - prior: List[Dict[str, Any]]) -> Dict[str, Any]: + prior: List[Dict[str, Any]], + window_truncated: bool = False) -> Dict[str, Any]: """ Decide whether a current failure is new, long-standing, never-working, or noise. + ``window_truncated`` says the caller could not see as far back as it asked + for. That only changes NEVER_PASSED: "has never passed" and "has not passed + within the few runs I could see" are different claims, and the second must + not be reported with high confidence. The other verdicts are unaffected -- + they are decided by the most recent entries, which are always present. + :param current: The current run's history entry, if it was found. :type current: Optional[Dict[str, Any]] :param prior: Entries for earlier runs, newest first. :type prior: List[Dict[str, Any]] + :param window_truncated: Whether older runs existed but could not be read. + :type window_truncated: bool :return: A verdict block with the supporting run ids and signature comparison. :rtype: Dict[str, Any] """ @@ -105,7 +114,8 @@ def classify_history(current: Optional[Dict[str, Any]], return {'verdict': NO_HISTORY, 'confidence': 'low', 'reason': 'No earlier run of this test to compare against', 'last_pass_run': None, 'previous_run': None, 'prior_runs_considered': 0, - 'transitions': 0, 'signature': signature, 'signature_changed': None} + 'transitions': 0, 'signature': signature, 'signature_changed': None, + 'window_truncated': window_truncated} previous = prior[0] last_pass = next((e for e in prior if _is_pass(e)), None) @@ -123,6 +133,7 @@ def classify_history(current: Optional[Dict[str, Any]], 'transitions': transitions, 'signature': signature, 'signature_changed': signature_changed, + 'window_truncated': window_truncated, } if transitions >= FLAKY_TRANSITION_THRESHOLD: @@ -132,6 +143,12 @@ def classify_history(current: Optional[Dict[str, Any]], elif _is_pass(previous): block.update(verdict=NEW_REGRESSION, confidence='high', reason=f"Passed in run {previous.get('run_id')}, fails here") + elif last_pass is None and window_truncated: + # Older runs exist but could not be read, so a pass may be just beyond + # the window. Reported as NEVER_PASSED for grouping, but not asserted. + block.update(verdict=NEVER_PASSED, confidence='low', + reason=(f'Has not passed in the {len(prior)} runs visible here, but older ' + f'runs could not be read, so it may have passed before that')) elif last_pass is None: block.update(verdict=NEVER_PASSED, confidence='high', reason=f'Has not passed in any of the last {len(prior)} runs') @@ -156,7 +173,8 @@ def unknown_history(reason: str) -> Dict[str, Any]: """ return {'verdict': UNKNOWN, 'confidence': 'low', 'reason': reason, 'last_pass_run': None, 'previous_run': None, 'prior_runs_considered': 0, - 'transitions': 0, 'signature': None, 'signature_changed': None} + 'transitions': 0, 'signature': None, 'signature_changed': None, + 'window_truncated': False} def group_by_verdict(failures: List[Dict[str, Any]]) -> Dict[str, int]: diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0d131e0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,36 @@ +"""Test-wide safety net for the saved login session. + +``sp auth login`` writes a bearer token to ``$XDG_CONFIG_HOME/sp/config.json``. +Without this fixture, any test that reaches that code path writes to the +developer's *real* config -- and it did: a test that omitted ``--no-save`` +replaced a live session with the fake token ``spci_x``, and because the +plaintext token is returned only once at creation, the real one was +unrecoverable. + +Relying on every future test to remember ``--no-save`` is what failed the first +time, so the isolation is applied automatically to every test rather than +opt-in. Tests that need to assert on the file (``tests/test_ux.py``) still +point ``XDG_CONFIG_HOME`` at their own temporary directory; overriding an +already-redirected variable is harmless. +""" + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_config_home(tmp_path, monkeypatch): + """ + Point ``XDG_CONFIG_HOME`` at a per-test temporary directory. + + :param tmp_path: pytest's per-test temporary directory. + :type tmp_path: pathlib.Path + :param monkeypatch: pytest's environment patcher, which restores on teardown. + :type monkeypatch: pytest.MonkeyPatch + """ + monkeypatch.setenv('XDG_CONFIG_HOME', str(tmp_path / 'config')) + # HOME too: config_path() falls back to ~/.config when XDG_CONFIG_HOME is + # unset, and a test that clears the environment would otherwise escape. + monkeypatch.setenv('HOME', str(tmp_path / 'home')) + os.makedirs(tmp_path / 'home', exist_ok=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6ceef4a..e65d968 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -276,8 +276,16 @@ def test_history_is_filtered_to_the_run_platform(self, mock_get, mock_paginated) @mock.patch('sp_cli.client.ApiClient.get_paginated') @mock.patch('sp_cli.client.ApiClient.get') - def test_history_depth_implies_the_flag_and_sets_the_page_size(self, mock_get, mock_paginated): - """--history-depth alone turns history on, and asks for depth + the current run.""" + def test_history_depth_implies_the_flag_and_asks_for_a_full_page(self, mock_get, mock_paginated): + """--history-depth alone turns history on; the page size is always the maximum. + + Sizing the page to depth + 1 looked right but delivered a fraction of + it: the endpoint returns entries for every regression test on the + sample and slices only afterwards, so the window has to be filtered + client-side out of as large a page as the API will give. + """ + from sp_cli.constants import MAX_PAGE_LIMIT + result = self._invoke(mock_get, mock_paginated, ['investigate', '9299', '--history-depth', '5'], {42: self._history_for(18), 43: self._history_for(137)}) @@ -285,8 +293,8 @@ def test_history_depth_implies_the_flag_and_sets_the_page_size(self, mock_get, m self.assertEqual(result.exit_code, 0) self.assertIn('by_verdict', json.loads(result.output)) history_call = next(c for c in mock_paginated.call_args_list if '/history' in c.args[0]) - self.assertEqual(history_call.kwargs['params']['limit'], 6) - self.assertEqual(history_call.kwargs['max_items'], 6) + self.assertEqual(history_call.kwargs['params']['limit'], MAX_PAGE_LIMIT) + self.assertEqual(history_call.kwargs['max_items'], MAX_PAGE_LIMIT) @mock.patch('sp_cli.client.ApiClient.get_paginated') @mock.patch('sp_cli.client.ApiClient.get') @@ -472,9 +480,12 @@ def test_auth_login_rejects_lifetime_over_the_api_cap(self, mock_request): def test_auth_login_sends_scopes_only_when_requested(self, mock_request): """Omitting --scope leaves the field out so the server picks its default set.""" mock_request.return_value = {'token': 'spci_x', 'token_name': 'sp-cli', 'scopes': []} + # --no-save: this asserts the request body, not the saved session. The + # conftest fixture makes it safe either way, but saying so locally keeps + # the next reader from wondering whether the write is intentional. result = self.runner.invoke(cli, ['auth', 'login', '--email', 'a@b.co', '--password', 'hunter22', '--scope', 'runs:read', - '--scope', 'results:read']) + '--scope', 'results:read', '--no-save']) self.assertEqual(result.exit_code, 0) body = mock_request.call_args.kwargs['json_body'] @@ -1080,3 +1091,274 @@ def test_the_run_itself_failing_is_still_fatal(self, mock_get, mock_paginated): result = self.runner.invoke(cli, ['investigate', '1', '--with-history']) self.assertEqual(result.exit_code, 4) + + +class HistoryDegradationOrderingTests(unittest.TestCase): + """The three interacting defects in the history circuit breaker. + + Each of these passed the original implementation's own tests, because those + only exercised one failing sample at a time. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + RUN = {'run_id': 9388, 'platform': 'linux', 'status': 'fail'} + SUMMARY = {'fail_count': 4, 'total_samples': 4, 'pass_count': 0} + + @staticmethod + def _rows(*pairs): + """Build failure rows as (sample_id, regression_test_id) pairs.""" + return [{'sample_id': s, 'regression_test_id': r, 'sample_name': f's{s}', + 'categories': [], 'status': 'fail', 'exit_code': 1, + 'expected_rc': 0, 'outputs': []} for s, r in pairs] + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_one_bad_sample_does_not_trip_the_breaker(self, mock_get, mock_paginated): + """A failure is remembered per sample, so shared samples are asked once.""" + # Sample 50 has three regression tests and is unreachable; 60 is fine. + rows = self._rows((50, 1), (50, 2), (50, 3), (60, 4)) + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [ + rows, + ApiError('connection_error', 'Read timed out.'), + [], # sample 60 resolves normally + ] + result = self.runner.invoke(cli, ['investigate', '9388', '--with-history']) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.stdout) + # Sample 50 asked once, not three times: 1 samples call + 2 history calls. + self.assertEqual(mock_paginated.call_count, 3) + self.assertFalse(report['history_incomplete']['gave_up']) + self.assertEqual(report['history_incomplete']['failed_samples'], [50]) + # Sample 60 still got a real verdict. + self.assertNotEqual(report['failures'][3]['history']['verdict'], 'UNKNOWN') + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_giving_up_still_uses_history_already_in_hand(self, mock_get, mock_paginated): + """A sample fetched successfully is classified even after the breaker trips.""" + # 42 succeeds first; 60/61/62 then fail and trip the breaker; 42 recurs. + rows = self._rows((42, 1), (60, 2), (61, 3), (62, 4), (42, 5)) + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [ + rows, + [], # sample 42 ok + ApiError('connection_error', 'boom'), + ApiError('connection_error', 'boom'), + ApiError('connection_error', 'boom'), + ] + result = self.runner.invoke(cli, ['investigate', '9388', '--with-history']) + + self.assertEqual(result.exit_code, 0) + report = json.loads(result.stdout) + self.assertTrue(report['history_incomplete']['gave_up']) + # The second row for sample 42 is classified from cache, not skipped. + last = report['failures'][4]['history'] + self.assertNotEqual(last['verdict'], 'UNKNOWN') + self.assertNotIn('not responding', str(last.get('reason', ''))) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_failed_samples_are_not_listed_twice(self, mock_get, mock_paginated): + """One unreachable sample appears once in the report, however many rows it has.""" + rows = self._rows((50, 1), (50, 2), (50, 3)) + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [rows, ApiError('connection_error', 'boom')] + result = self.runner.invoke(cli, ['investigate', '9388', '--with-history']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual( + json.loads(result.stdout)['history_incomplete']['failed_samples'], [50]) + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + def test_history_depth_is_bounded_by_the_api_page_limit(self, mock_paginated): + """limit = depth + 1 must stay <= 100, or every lookup 400s.""" + for bad in ('100', '500', '0', '-1'): + result = self.runner.invoke(cli, ['investigate', '9388', '--history-depth', bad]) + self.assertNotEqual(result.exit_code, 0, f'--history-depth {bad} should be rejected') + mock_paginated.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get_paginated') + @mock.patch('sp_cli.client.ApiClient.get') + def test_the_largest_accepted_depth_still_fits_the_page(self, mock_get, mock_paginated): + """depth 99 sends limit 100, which is exactly the API's ceiling.""" + mock_get.side_effect = [self.RUN, self.SUMMARY] + mock_paginated.side_effect = [self._rows((42, 1)), []] + result = self.runner.invoke(cli, ['investigate', '9388', '--history-depth', '99']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(mock_paginated.call_args.kwargs['params']['limit'], 100) + + +class TruncatedOutputTests(unittest.TestCase): + """`--decode` must not write a file that ends mid-stream and looks complete. + + The API inlines at most 1 MiB and flags the rest as truncated; writing the + fragment made every diff against it report spurious missing lines. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + DETAIL = {'sample_id': 11, 'regression_test_id': 11, + 'outputs': [{'output_id': 11, 'status': 'fail'}]} + + BIG = {'content': 'dHJ1bmNhdGVk', 'encoding': 'base64', 'truncated': True, + 'sha256': 'abc123', 'download_url': 'https://storage.example/whole-file'} + + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_truncated_output_is_refused_not_silently_written(self, mock_get): + """Nothing reaches stdout, and the error names where to get the whole file.""" + mock_get.side_effect = [self.DETAIL, self.BIG] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11', '--decode']) + + self.assertNotEqual(result.exit_code, 0) + self.assertEqual(result.stdout_bytes, b'') + envelope = json.loads(result.stderr) + self.assertEqual(envelope['error']['code'], 'output_truncated') + self.assertEqual(envelope['error']['details']['download_url'], + 'https://storage.example/whole-file') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_allow_truncated_opts_in_explicitly(self, mock_get): + """The capability is kept, but you have to ask for it by name.""" + mock_get.side_effect = [self.DETAIL, self.BIG] + result = self.runner.invoke( + cli, ['run', 'output', '9388', '11', '--decode', '--allow-truncated']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.stdout_bytes, b'truncated') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_complete_output_is_unaffected(self, mock_get): + """The ordinary path still writes the file.""" + mock_get.side_effect = [ + self.DETAIL, + {'content': 'aGVsbG8=', 'encoding': 'base64', 'truncated': False}, + ] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11', '--decode']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.stdout_bytes, b'hello') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_missing_truncated_key_is_treated_as_complete(self, mock_get): + """Older responses without the field must not start failing.""" + mock_get.side_effect = [self.DETAIL, {'content': 'aGk=', 'encoding': 'base64'}] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11', '--decode']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.stdout_bytes, b'hi') + + @mock.patch('sp_cli.client.ApiClient.get') + def test_the_json_envelope_still_shows_truncation(self, mock_get): + """Without --decode the flag is visible to the caller as data.""" + mock_get.side_effect = [self.DETAIL, self.BIG] + result = self.runner.invoke(cli, ['run', 'output', '9388', '11']) + + self.assertEqual(result.exit_code, 0) + self.assertTrue(json.loads(result.stdout)['truncated']) + + +class MachineOutputContractTests(unittest.TestCase): + """Every command's stdout must be parseable JSON in the default mode. + + `auth revoke` and `auth logout` printed bare English sentences, so piping + them into jq failed while every other command worked. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_auth_revoke_emits_json(self, mock_request): + """`sp auth revoke 5 | jq .` must not choke on prose.""" + mock_request.return_value = None + result = self.runner.invoke(cli, ['auth', 'revoke', '5']) + + self.assertEqual(result.exit_code, 0) + payload = json.loads(result.stdout) + self.assertEqual(payload, {'token_id': 5, 'revoked': True}) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_auth_logout_emits_json(self, mock_request): + """Same for logout, including whether the saved session went with it.""" + mock_request.return_value = None + result = self.runner.invoke(cli, ['auth', 'logout']) + + self.assertEqual(result.exit_code, 0) + payload = json.loads(result.stdout) + self.assertIs(payload['revoked'], True) + self.assertIn('saved_session_cleared', payload) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_no_command_leaks_prose_onto_stdout(self, mock_request): + """A sweep over the write commands that answer 204, which have no body to render.""" + mock_request.return_value = None + for args in (['auth', 'revoke', '5'], ['auth', 'logout']): + result = self.runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, args) + try: + json.loads(result.stdout) + except ValueError: # pragma: no cover - the assertion message is the point + self.fail(f'{args} wrote non-JSON to stdout: {result.stdout!r}') + + +class PageLimitContractTests(unittest.TestCase): + """--limit and --offset are bounded by what the API's paginator accepts. + + LOG_MAX_LIMIT mirrored a 1-500 clamp inside the log service, but + _parse_limit rejects anything over 100 first, so that clamp is unreachable + and the CLI advertised a page size the API refuses. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_run_logs_limit_is_capped_at_the_paginator_ceiling(self, mock_get): + """The value the old help text advertised as legal is now rejected locally.""" + result = self.runner.invoke(cli, ['run', 'logs', '9299', '--limit', '500']) + + self.assertNotEqual(result.exit_code, 0) + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_the_ceiling_itself_is_accepted(self, mock_get): + """100 is legal; 101 is not.""" + mock_get.return_value = {'data': [], 'pagination': {}} + self.assertEqual( + self.runner.invoke(cli, ['run', 'logs', '9299', '--limit', '100']).exit_code, 0) + self.assertNotEqual( + self.runner.invoke(cli, ['run', 'logs', '9299', '--limit', '101']).exit_code, 0) + + @mock.patch('sp_cli.client.ApiClient.get') + def test_every_paginated_command_rejects_an_out_of_range_limit(self, mock_get): + """One shared rule, so it is checked across the surface rather than per command.""" + commands = ( + ['run', 'ls'], ['run', 'results', '9299'], ['run', 'errors', '9299'], + ['run', 'artifacts', '9299'], ['run', 'progress', '9299'], + ['sample', 'ls'], ['sample', 'history', '42'], ['regression', 'ls'], + ['category', 'ls'], ['auth', 'tokens'], ['auth', 'users'], ['queue'], + ['admin', 'blocked-users', 'ls'], ['admin', 'forbidden-extensions', 'ls'], + ) + for cmd in commands: + for bad in ('0', '101'): + result = self.runner.invoke(cli, cmd + ['--limit', bad]) + self.assertNotEqual(result.exit_code, 0, f'{cmd} --limit {bad} should be rejected') + mock_get.assert_not_called() + + @mock.patch('sp_cli.client.ApiClient.get') + def test_a_negative_offset_is_rejected(self, mock_get): + """The API 400s on a negative offset; fail locally instead.""" + result = self.runner.invoke(cli, ['run', 'ls', '--offset', '-1']) + + self.assertNotEqual(result.exit_code, 0) + mock_get.assert_not_called() diff --git a/tests/test_history.py b/tests/test_history.py index 374608d..5f63204 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -163,3 +163,58 @@ def test_unknown_history_has_the_same_shape_as_a_real_verdict(self): real = classify_history(entry(9299, 'fail'), [entry(9298, 'pass')]) self.assertEqual(set(unknown_history('no id')), set(real)) + + +class TruncatedWindowTests(unittest.TestCase): + """NEVER_PASSED must not be asserted confidently on a window we could not see past. + + /samples/{id}/history returns entries for every regression test on the + sample and slices only afterwards, so the window for one test is roughly + the page size divided by how many tests share the sample. A test that + passed nine runs ago looked like it had never passed at all. + """ + + @staticmethod + def _fails(*run_ids): + """Build failing prior entries, newest first.""" + return [{'run_id': r, 'status': 'fail', 'regression_test_id': 1} for r in run_ids] + + def test_a_short_window_downgrades_confidence(self): + """Same verdict for grouping, but not claimed as fact.""" + block = classify_history(None, self._fails(98, 97), window_truncated=True) + + self.assertEqual(block['verdict'], NEVER_PASSED) + self.assertEqual(block['confidence'], 'low') + self.assertTrue(block['window_truncated']) + self.assertIn('older runs could not be read', block['reason']) + + def test_a_complete_window_still_asserts_it(self): + """When the window really is the whole history, the claim stands.""" + block = classify_history(None, self._fails(98, 97), window_truncated=False) + + self.assertEqual(block['verdict'], NEVER_PASSED) + self.assertEqual(block['confidence'], 'high') + self.assertFalse(block['window_truncated']) + + def test_truncation_does_not_weaken_the_other_verdicts(self): + """NEW_REGRESSION and STILL_FAILING are decided by the newest entries, always present.""" + passed_then_failed = [{'run_id': 98, 'status': 'pass', 'regression_test_id': 1}] + new = classify_history(None, passed_then_failed, window_truncated=True) + self.assertEqual(new['verdict'], NEW_REGRESSION) + self.assertEqual(new['confidence'], 'high') + + with_a_pass = self._fails(98) + [{'run_id': 97, 'status': 'pass', + 'regression_test_id': 1}] + still = classify_history(None, with_a_pass, window_truncated=True) + self.assertEqual(still['verdict'], STILL_FAILING) + self.assertEqual(still['confidence'], 'high') + + def test_every_verdict_block_carries_the_flag(self): + """Uniform shape keeps the JSON safe to iterate over.""" + blocks = [ + classify_history(None, [], window_truncated=True), + classify_history(None, self._fails(98), window_truncated=True), + unknown_history('lookup failed'), + ] + for block in blocks: + self.assertIn('window_truncated', block, block['verdict']) diff --git a/tests/test_ux.py b/tests/test_ux.py index cc58e45..1f493cd 100644 --- a/tests/test_ux.py +++ b/tests/test_ux.py @@ -251,3 +251,152 @@ def test_a_disabled_spinner_writes_nothing(self): with Spinner('working', enabled=True): pass mock_write.assert_not_called() + + +class ConfigIsolationTests(unittest.TestCase): + """The suite must never be able to touch the developer's real config. + + A test that omitted --no-save replaced a live saved session with the fake + token `spci_x`. The plaintext token is returned only once at creation, so + the real credential was unrecoverable. The conftest fixture makes that + impossible; this asserts the fixture is actually in force. + """ + + def test_config_path_is_redirected_away_from_the_real_home(self): + """XDG_CONFIG_HOME must point somewhere disposable during tests.""" + path = config.config_path() + + self.assertNotEqual(path, Path.home() / '.config' / 'sp' / 'config.json') + self.assertNotIn('/.config/sp/config.json', str(Path('~').expanduser() / '.config')) + # And the redirect target must not be the real user's home. + self.assertFalse(str(path).startswith(str(Path('~').expanduser()) + '/.config/sp')) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_a_login_without_no_save_cannot_escape_the_sandbox(self, mock_request): + """Even a careless future test writes only inside the fixture's tmp dir.""" + mock_request.return_value = {'token': 'spci_careless'} + result = CliRunner().invoke(cli, [ + 'auth', 'login', '--email', 'a@b.c', '--password', 'pw']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(config.saved_token(), 'spci_careless') + # Written, but under the redirected home -- not the real one. + self.assertTrue(str(config.config_path()).startswith(os.environ['XDG_CONFIG_HOME'])) + + +class LogoutOwnershipTests(unittest.TestCase): + """`logout` must only clear a session it actually owns. + + Revoking a scratch token from --token/SP_API_TOKEN used to delete an + unrelated saved credential, which cannot be recovered because the plaintext + token is returned only once at creation. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + @mock.patch('sp_cli.client.ApiClient.request') + def test_revoking_a_scratch_token_leaves_the_saved_session_alone(self, mock_request): + """The saved 30-day session survives `SP_API_TOKEN=... sp auth logout`.""" + config.save_token('the-real-session') + mock_request.return_value = None + + with mock.patch.dict(os.environ, {'SP_API_TOKEN': 'scratch-token'}): + result = self.runner.invoke(cli, ['auth', 'logout']) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(config.saved_token(), 'the-real-session') + self.assertFalse(json.loads(result.stdout)['saved_session_cleared']) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_revoking_the_saved_token_does_clear_it(self, mock_request): + """The ordinary case still logs you out.""" + config.save_token('the-real-session') + mock_request.return_value = None + + with mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop('SP_API_TOKEN', None) + result = self.runner.invoke(cli, ['auth', 'logout']) + + self.assertEqual(result.exit_code, 0) + self.assertIsNone(config.saved_token()) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_an_explicit_token_matching_the_saved_one_still_clears(self, mock_request): + """Matched by value, not provenance, so --token with the same value counts.""" + config.save_token('same-token') + mock_request.return_value = None + + result = self.runner.invoke(cli, ['--token', 'same-token', 'auth', 'logout']) + + self.assertEqual(result.exit_code, 0) + self.assertIsNone(config.saved_token()) + + @mock.patch('sp_cli.client.ApiClient.request') + def test_a_dropped_connection_does_not_discard_a_good_token(self, mock_request): + """The server was never reached; the token may still be perfectly valid.""" + from sp_cli.client import ApiError + config.save_token('probably-fine') + mock_request.side_effect = ApiError('connection_error', 'Could not reach host') + + result = self.runner.invoke(cli, ['auth', 'logout']) + + self.assertEqual(result.exit_code, 3) + self.assertEqual(config.saved_token(), 'probably-fine') + + @mock.patch('sp_cli.client.ApiClient.request') + def test_a_rejected_token_is_still_cleared(self, mock_request): + """401 means it cannot work again, so leaving it on disk helps nobody.""" + from sp_cli.client import ApiError + config.save_token('expired') + mock_request.side_effect = ApiError('unauthorized', 'Token expired.', 401) + + result = self.runner.invoke(cli, ['auth', 'logout']) + + self.assertEqual(result.exit_code, 6) + self.assertIsNone(config.saved_token()) + + +class TokenFilePermissionTests(unittest.TestCase): + """The token must never be written into a file others can read. + + os.open's mode applies only when the file is created, so a re-login into an + already-loose file used to write the secret first and chmod second. + """ + + def setUp(self): + """Create a runner for each test.""" + self.runner = CliRunner() + + def test_rewriting_a_loose_file_tightens_it_before_writing(self): + """The permissions are 0600 by the time any token byte is on disk.""" + path = config.save_token('first-token') + os.chmod(path, 0o644) + self.assertTrue(config.is_world_readable()) + + observed = [] + real_dump = json.dump + + def spy(data, handle, **kwargs): + # Snapshot the mode at the moment the secret is being serialized. + observed.append(stat.S_IMODE(os.fstat(handle.fileno()).st_mode)) + return real_dump(data, handle, **kwargs) + + with mock.patch('sp_cli.config.json.dump', spy): + config.save_token('second-token') + + self.assertEqual(observed, [0o600], + f'token written while mode was {[oct(m) for m in observed]}') + self.assertEqual(stat.S_IMODE(Path(path).stat().st_mode), 0o600) + self.assertFalse(config.is_world_readable()) + + def test_clear_token_also_rewrites_privately(self): + """The same path is used when logout rewrites the remaining settings.""" + path = config.save_token('a-token', 'http://example.test/api/v1') + os.chmod(path, 0o644) + + config.clear_token() + + self.assertEqual(stat.S_IMODE(Path(path).stat().st_mode), 0o600) + self.assertEqual(config.load().get('base_url'), 'http://example.test/api/v1')