From bb627a8d6ca7967611809e884ca950d6da5e16a1 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Fri, 31 Jul 2026 23:53:08 +0530 Subject: [PATCH 1/7] feat(cli): add environment readiness check Add a CI-friendly hud check command that can launch and poll active readiness probes with stable exit semantics. Co-authored-by: Cursor --- hud/cli/__init__.py | 2 + hud/cli/check.py | 147 ++++++++++++++++++++++++ hud/cli/tests/test_check.py | 215 ++++++++++++++++++++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 hud/cli/check.py create mode 100644 hud/cli/tests/test_check.py diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index ed4f6e1d1..0860f0735 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -31,6 +31,7 @@ # --------------------------------------------------------------------------- from .cancel import cancel_command # noqa: E402 +from .check import check_command # noqa: E402 from .client import client_app # noqa: E402 from .deploy import deploy_command # noqa: E402 from .eval import eval_command # noqa: E402 @@ -50,6 +51,7 @@ app.command(name="eval")(eval_command) app.command(name="init")(init_command) app.command(name="cancel")(cancel_command) +app.command(name="check")(check_command) app.add_typer(models_app, name="models") app.add_typer(jobs_app, name="jobs") app.add_typer(trace_app, name="trace") diff --git a/hud/cli/check.py b/hud/cli/check.py new file mode 100644 index 000000000..34a265208 --- /dev/null +++ b/hud/cli/check.py @@ -0,0 +1,147 @@ +"""Run the platform's environment readiness checklist.""" + +from __future__ import annotations + +import json +import time +from typing import Any, cast + +import typer + +from hud.cli.utils.api import require_api_key +from hud.cli.utils.registry import resolve_registry_environments +from hud.utils.exceptions import HudRequestError +from hud.utils.platform import PlatformClient + +_POLL_INTERVAL_SECONDS = 2.0 + + +def _request_error(message: str) -> None: + typer.echo(message, err=True) + raise typer.Exit(2) + + +def _resolve_environment_id(platform: PlatformClient, reference: str) -> str: + matches = resolve_registry_environments(platform, reference) + if not matches: + _request_error(f"No environment matched {reference!r}.") + if len(matches) > 1: + names = ", ".join(f"{match.name} ({match.short_id})" for match in matches) + _request_error(f"{reference!r} matches multiple environments: {names}") + return matches[0].id + + +def _probe_is_running(report: dict[str, Any]) -> bool: + criteria = report.get("criteria") + if not isinstance(criteria, list): + return False + for raw_criterion in cast("list[object]", criteria): + if not isinstance(raw_criterion, dict): + continue + criterion = cast("dict[str, Any]", raw_criterion) + if criterion.get("check_key") != "environment_probe_task_runs": + continue + evidence = criterion.get("evidence") + return ( + criterion.get("status") == "unknown" + and isinstance(evidence, dict) + and isinstance(cast("dict[str, Any]", evidence).get("execution_trace_id"), str) + ) + return False + + +def check_command( + environment: str = typer.Argument(..., help="Environment name or UUID."), + json_output: bool = typer.Option(False, "--json", help="Output the machine-readable report."), + overwrite: bool = typer.Option( + False, + "--overwrite", + help="Create a fresh check attempt instead of reusing an identical result.", + ), + active: bool = typer.Option( + False, + "--active", + help="Start the declared worker-backed readiness probe.", + ), + model: str | None = typer.Option( + None, + "--model", + help="Probe model override; otherwise use the platform default.", + ), + wait: bool = typer.Option( + True, + "--wait/--no-wait", + help="Poll an active probe until its trace reaches a terminal state.", + ), + timeout: float = typer.Option( + 600, + "--timeout", + min=1, + help="Maximum seconds to wait for an active probe.", + ), +) -> None: + """Check whether a deployed HUD environment is ready for task execution.""" + try: + require_api_key("check an environment") + except typer.Exit as exc: + raise typer.Exit(2) from exc + + platform = PlatformClient.from_settings() + if model is not None and not active: + _request_error("--model requires --active") + try: + environment_id = _resolve_environment_id(platform, environment) + request: dict[str, object] = { + "environment_id": environment_id, + "overwrite": overwrite, + } + if active: + request["run_active_probe"] = True + if model is not None: + request["probe_model"] = model + raw_report = platform.post( + "/checks/environment-readiness", + json=request, + ) + except HudRequestError as exc: + _request_error(str(exc)) + + if not isinstance(raw_report, dict): + typer.echo("Platform returned an invalid readiness report.", err=True) + raise typer.Exit(3) + report = cast("dict[str, Any]", raw_report) + deadline = time.monotonic() + timeout + while active and wait and _probe_is_running(report) and time.monotonic() < deadline: + time.sleep(_POLL_INTERVAL_SECONDS) + poll_request = {**request, "overwrite": False} + try: + raw_report = platform.post("/checks/environment-readiness", json=poll_request) + except HudRequestError as exc: + _request_error(str(exc)) + if not isinstance(raw_report, dict): + typer.echo("Platform returned an invalid readiness report.", err=True) + raise typer.Exit(3) + report = cast("dict[str, Any]", raw_report) + poll_timed_out = active and wait and _probe_is_running(report) + status = report.get("status") + if json_output: + typer.echo(json.dumps(report, indent=2, sort_keys=True, default=str)) + else: + human_report = report.get("human_report") + typer.echo( + human_report + if isinstance(human_report, str) + else str(report.get("summary") or "No readiness summary returned."), + ) + + if poll_timed_out: + typer.echo(f"Timed out after {timeout:g}s waiting for the readiness probe.", err=True) + raise typer.Exit(3) + if status == "passed": + return + if status == "error": + raise typer.Exit(3) + if status in {"failed", "unknown"}: + raise typer.Exit(1) + typer.echo(f"Platform returned unknown readiness status: {status!r}", err=True) + raise typer.Exit(3) diff --git a/hud/cli/tests/test_check.py b/hud/cli/tests/test_check.py new file mode 100644 index 000000000..fef804a7d --- /dev/null +++ b/hud/cli/tests/test_check.py @@ -0,0 +1,215 @@ +"""CLI behavior for HUD environment readiness checks.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from hud.cli import app +from hud.cli.utils.registry import RegistryEnvironment + +runner = CliRunner() + + +def _report(status: str) -> dict[str, object]: + return { + "schema_version": "environment_readiness.v1", + "environment_id": "environment-id", + "status": status, + "ready": status == "passed", + "summary": f"Environment is {status}.", + "human_report": f"Environment readiness: {status.upper()}", + "criteria": [], + } + + +def test_check_prints_report_and_exits_one_when_not_ready() -> None: + """A completed advisory failure is distinct from a request error.""" + platform = MagicMock() + platform.post.return_value = _report("failed") + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[RegistryEnvironment(id="environment-id", name="browser")], + ), + ): + result = runner.invoke(app, ["check", "browser"]) + + assert result.exit_code == 1 + assert "Environment readiness: FAILED" in result.output + platform.post.assert_called_once_with( + "/checks/environment-readiness", + json={"environment_id": "environment-id", "overwrite": False}, + ) + + +def test_check_json_preserves_machine_contract_and_success_exit() -> None: + """JSON output is stable enough for CI and returns zero only when ready.""" + platform = MagicMock() + platform.post.return_value = _report("passed") + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[RegistryEnvironment(id="environment-id", name="browser")], + ), + ): + result = runner.invoke(app, ["check", "browser", "--json", "--overwrite"]) + + assert result.exit_code == 0 + assert json.loads(result.output)["status"] == "passed" + platform.post.assert_called_once_with( + "/checks/environment-readiness", + json={"environment_id": "environment-id", "overwrite": True}, + ) + + +def test_check_rejects_ambiguous_environment_without_running() -> None: + """Name resolution must not silently select the wrong environment.""" + platform = MagicMock() + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[ + RegistryEnvironment(id="first-id", name="browser"), + RegistryEnvironment(id="second-id", name="browser-dev"), + ], + ), + ): + result = runner.invoke(app, ["check", "browser"]) + + assert result.exit_code == 2 + assert "matches multiple environments" in result.output + platform.post.assert_not_called() + + +def test_check_returns_three_for_platform_execution_error() -> None: + """A completed platform execution error has its documented exit code.""" + platform = MagicMock() + platform.post.return_value = _report("error") + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[RegistryEnvironment(id="environment-id", name="browser")], + ), + ): + result = runner.invoke(app, ["check", "browser"]) + + assert result.exit_code == 3 + + +def test_check_active_requests_worker_probe_with_model_override() -> None: + """The opt-in active mode forwards execution intent without changing static calls.""" + platform = MagicMock() + platform.post.return_value = _report("unknown") + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[RegistryEnvironment(id="environment-id", name="browser")], + ), + ): + result = runner.invoke( + app, + ["check", "browser", "--active", "--model", "probe-model"], + ) + + assert result.exit_code == 1 + platform.post.assert_called_once_with( + "/checks/environment-readiness", + json={ + "environment_id": "environment-id", + "overwrite": False, + "run_active_probe": True, + "probe_model": "probe-model", + }, + ) + + +def test_check_active_polls_until_probe_is_terminal() -> None: + """Polling reuses the running attempt even when the initial request overwrites.""" + running = _report("unknown") + running["criteria"] = [ + { + "check_key": "environment_probe_task_runs", + "status": "unknown", + "evidence": {"execution_trace_id": "trace-id"}, + }, + ] + completed = _report("unknown") + completed["criteria"] = [ + { + "check_key": "environment_probe_task_runs", + "status": "passed", + "evidence": {"execution_trace_id": "trace-id"}, + }, + ] + platform = MagicMock() + platform.post.side_effect = [running, completed] + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[RegistryEnvironment(id="environment-id", name="browser")], + ), + patch("hud.cli.check.time.sleep") as sleep, + ): + result = runner.invoke( + app, + ["check", "browser", "--active", "--overwrite"], + ) + + assert result.exit_code == 1 + sleep.assert_called_once() + assert platform.post.call_count == 2 + assert platform.post.call_args_list[0].kwargs["json"]["overwrite"] is True + assert platform.post.call_args_list[1].kwargs["json"]["overwrite"] is False + + +def test_check_active_poll_timeout_is_execution_error() -> None: + """A locally exhausted wait budget is not a completed quality verdict.""" + running = _report("unknown") + running["criteria"] = [ + { + "check_key": "environment_probe_task_runs", + "status": "unknown", + "evidence": {"execution_trace_id": "trace-id"}, + }, + ] + platform = MagicMock() + platform.post.return_value = running + + with ( + patch("hud.cli.check.require_api_key", return_value="api-key"), + patch("hud.cli.check.PlatformClient.from_settings", return_value=platform), + patch( + "hud.cli.check.resolve_registry_environments", + return_value=[RegistryEnvironment(id="environment-id", name="browser")], + ), + patch("hud.cli.check.time.monotonic", side_effect=[0, 2]), + ): + result = runner.invoke( + app, + ["check", "browser", "--active", "--timeout", "1"], + ) + + assert result.exit_code == 3 + assert "Timed out after 1s" in result.output + platform.post.assert_called_once() From dede9adecb112305dd5f7cad3d1cb63841bd2f04 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Sun, 2 Aug 2026 10:50:31 +0530 Subject: [PATCH 2/7] feat(environment): publish readiness declarations Validate non-secret V0 readiness contracts at authoring time and include them in control-channel hello manifests for build introspection. Co-authored-by: Cursor --- hud/clients/client.py | 4 ++ hud/environment/__init__.py | 3 +- hud/environment/env.py | 97 +++++++++++++++++++++++++++- hud/environment/server.py | 5 ++ hud/environment/tests/test_server.py | 42 +++++++++++- 5 files changed, 147 insertions(+), 4 deletions(-) diff --git a/hud/clients/client.py b/hud/clients/client.py index dd59c4d13..78cf63ddc 100644 --- a/hud/clients/client.py +++ b/hud/clients/client.py @@ -74,6 +74,7 @@ class Manifest: protocol_version: str # e.g. "hud/1.0" server_info: ServerInfo bindings: list[Capability] + readiness: dict[str, Any] | None = None class HudClient: @@ -158,6 +159,9 @@ async def hello(self, session_id: str | None = None) -> Manifest: version=env.get("version", "0.0.0"), ), bindings=bindings, + readiness=( + result["readiness"] if isinstance(result.get("readiness"), dict) else None + ), ) return self.manifest diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index 002eef168..f17503962 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -22,7 +22,7 @@ from hud.capabilities import Capability from hud.utils.modules import iter_modules -from .env import Answer, Environment +from .env import Answer, Environment, EnvironmentReadiness from .integration import Integration from .workspace import DEFAULT_SYSTEM_MOUNTS, Mount, MountKind, Workspace @@ -92,6 +92,7 @@ def load_environment( "Answer", "Capability", "Environment", + "EnvironmentReadiness", "Integration", "Mount", "MountKind", diff --git a/hud/environment/env.py b/hud/environment/env.py index cc5d96327..571ed4c8a 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -10,10 +10,11 @@ import contextlib import functools import inspect +import re from contextvars import ContextVar -from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, TypeVar, cast -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, create_model +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, create_model, field_validator from hud.capabilities import Capability @@ -48,6 +49,94 @@ class Answer(BaseModel, Generic[T]): raw: str = Field(default="", description="Original answer string before parsing") +_READINESS_SECRET_KEYS = frozenset( + { + "api_key", + "access_token", + "authorization", + "client_secret", + "credential", + "credentials", + "hud_api_key", + "mcp_config", + "password", + "private_key", + "refresh_token", + "secret", + "secrets", + "token", + } +) +_READINESS_SECRET_SUFFIXES = ( + "_access_key", + "_api_key", + "_client_secret", + "_credential", + "_credentials", + "_password", + "_private_key", + "_secret", + "_secret_access_key", + "_token", +) + + +def _readiness_has_secret_key(value: Any) -> bool: + if isinstance(value, dict): + for key, item in cast("dict[str, Any]", value).items(): + snake = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", key) + normalized = re.sub(r"[^a-z0-9]+", "_", snake.lower()).strip("_") + if ( + normalized in _READINESS_SECRET_KEYS + or normalized.endswith(_READINESS_SECRET_SUFFIXES) + or _readiness_has_secret_key(item) + ): + return True + return False + if isinstance(value, list): + return any(_readiness_has_secret_key(item) for item in cast("list[Any]", value)) + return False + + +class _EnvironmentReadinessProbe(BaseModel): + model_config = ConfigDict(extra="forbid") + + scenario: str = Field(min_length=1) + args: dict[str, Any] = Field(default_factory=dict) + + @field_validator("args") + @classmethod + def _reject_secrets(cls, value: dict[str, Any]) -> dict[str, Any]: + if _readiness_has_secret_key(value): + raise ValueError("readiness probe args must not contain secret-bearing keys") + return value + + +class _EnvironmentReadinessReset(BaseModel): + model_config = ConfigDict(extra="forbid") + + strategy: Literal["reprovision"] + + +class _EnvironmentReadinessBudgets(BaseModel): + model_config = ConfigDict(extra="forbid") + + startup_timeout_s: int = Field(gt=0) + probe_timeout_s: int = Field(gt=0) + reset_timeout_s: int = Field(gt=0) + + +class EnvironmentReadiness(BaseModel): + """Versioned active-probe declaration published in the environment manifest.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["hud.environment-readiness.v0"] + probe: _EnvironmentReadinessProbe + reset: _EnvironmentReadinessReset + budgets: _EnvironmentReadinessBudgets + + def _args_json_schema(sig: inspect.Signature) -> dict[str, Any]: """JSON Schema for a task function's parameters — the task's args contract. @@ -142,6 +231,7 @@ def __init__( *, version: str = "0.0.1", capabilities: Sequence[Capability] | None = None, + readiness: EnvironmentReadiness | dict[str, Any] | None = None, **legacy_kwargs: Any, ) -> None: if legacy_kwargs: @@ -155,6 +245,9 @@ def __init__( ) self.name = name self.version = version + self.readiness = ( + EnvironmentReadiness.model_validate(readiness) if readiness is not None else None + ) #: Published capabilities — always concrete wire data. Daemons the env #: runs itself publish theirs at serve time (:meth:`add_capability` #: from an ``@env.initialize`` hook; :meth:`workspace` wires the diff --git a/hud/environment/server.py b/hud/environment/server.py index 1e578a327..3aa0f7b05 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -332,6 +332,11 @@ async def error_to(msg_id: int | None, code: int, message: str) -> None: "session_id": session_id, "env": {"name": env.name, "version": env.version}, "bindings": bindings, + "readiness": ( + env.readiness.model_dump(mode="json") + if env.readiness is not None + else None + ), }, ) diff --git a/hud/environment/tests/test_server.py b/hud/environment/tests/test_server.py index f615f5115..993d7ce7d 100644 --- a/hud/environment/tests/test_server.py +++ b/hud/environment/tests/test_server.py @@ -10,7 +10,7 @@ from typing import Literal import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError from hud.clients import HudProtocolError from hud.environment import Answer, Environment @@ -27,6 +27,46 @@ class _Payload(BaseModel): _Mode = Literal["upper", "lower"] +async def test_hello_publishes_validated_readiness_declaration() -> None: + """The author-facing declaration is emitted on the control-channel hello frame.""" + readiness = { + "schema_version": "hud.environment-readiness.v0", + "probe": {"scenario": "probe", "args": {"seed": 0}}, + "reset": {"strategy": "reprovision"}, + "budgets": { + "startup_timeout_s": 120, + "probe_timeout_s": 300, + "reset_timeout_s": 180, + }, + } + env = Environment("ready", readiness=readiness) + + async with served(env) as client: + assert client.manifest is not None + assert client.manifest.readiness == readiness + + +def test_readiness_declaration_rejects_secret_args() -> None: + """Environment authors cannot publish credentials as readiness arguments.""" + with pytest.raises(ValidationError, match="must not contain"): + Environment( + "unsafe", + readiness={ + "schema_version": "hud.environment-readiness.v0", + "probe": { + "scenario": "probe", + "args": {"nested": {"apiKey": "secret"}}, + }, + "reset": {"strategy": "reprovision"}, + "budgets": { + "startup_timeout_s": 120, + "probe_timeout_s": 300, + "reset_timeout_s": 180, + }, + }, + ) + + async def test_dict_grade_without_numeric_score_errors_loudly() -> None: env = Environment("badgrade") From 61ca5438fd3c1b46ad8824791228d311ae1633ab Mon Sep 17 00:00:00 2001 From: CK0607 Date: Sun, 2 Aug 2026 13:48:40 +0530 Subject: [PATCH 3/7] feat(cli): add resource QA agent commands Expose agent discovery, run, wait, and result inspection for canonical Environment and Taskset subjects so QA workflows no longer require direct API calls. Co-authored-by: Cursor --- hud/cli/__init__.py | 2 + hud/cli/qa.py | 268 +++++++++++++++++++++++++++++++ hud/cli/tests/test_qa.py | 244 ++++++++++++++++++++++++++++ hud/clients/client.py | 4 +- hud/utils/platform.py | 2 +- hud/utils/tests/test_platform.py | 13 ++ 6 files changed, 529 insertions(+), 4 deletions(-) create mode 100644 hud/cli/qa.py create mode 100644 hud/cli/tests/test_qa.py diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index 0860f0735..f01a6a986 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -39,6 +39,7 @@ from .jobs import jobs_app # noqa: E402 from .login import login_command # noqa: E402 from .models import models_app # noqa: E402 +from .qa import qa_app # noqa: E402 from .serve import serve_command # noqa: E402 from .sync import sync_app # noqa: E402 from .task import task_app # noqa: E402 @@ -55,6 +56,7 @@ app.add_typer(models_app, name="models") app.add_typer(jobs_app, name="jobs") app.add_typer(trace_app, name="trace") +app.add_typer(qa_app, name="qa") @app.command(name="set") diff --git a/hud/cli/qa.py b/hud/cli/qa.py new file mode 100644 index 000000000..210acdbcd --- /dev/null +++ b/hud/cli/qa.py @@ -0,0 +1,268 @@ +"""Discover, run, and inspect resource-scoped platform QA agents.""" + +from __future__ import annotations + +import json +import time +from typing import Any, cast + +import typer + +from hud.cli.utils.api import require_api_key +from hud.utils.exceptions import HudRequestError +from hud.utils.platform import PlatformClient + +_POLL_INTERVAL_SECONDS = 2.0 +_RESOURCE_SUBJECT_TYPES = {"environment", "taskset"} +_TERMINAL_STATUSES = {"completed", "error"} + +qa_app = typer.Typer( + name="qa", + help="Discover, run, and inspect platform QA agents.", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +def _request_error(message: str) -> None: + typer.echo(message, err=True) + raise typer.Exit(2) + + +def _platform() -> PlatformClient: + try: + require_api_key("use platform QA agents") + except typer.Exit as exc: + raise typer.Exit(2) from exc + return PlatformClient.from_settings() + + +def _subject_type(value: str) -> str: + normalized = value.strip().lower() + if normalized not in _RESOURCE_SUBJECT_TYPES: + choices = ", ".join(sorted(_RESOURCE_SUBJECT_TYPES)) + _request_error(f"Subject type must be one of: {choices}.") + return normalized + + +def _dict_list(value: object, *, label: str) -> list[dict[str, Any]]: + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + typer.echo(f"Platform returned invalid {label}.", err=True) + raise typer.Exit(3) + return cast("list[dict[str, Any]]", value) + + +def _print_json(value: object) -> None: + typer.echo(json.dumps(value, indent=2, sort_keys=True, default=str)) + + +def _result_verdict(result: dict[str, Any]) -> tuple[str, str | None]: + canonical = result.get("canonical_result") + if isinstance(canonical, dict): + canonical_dict = cast("dict[str, Any]", canonical) + verdict = canonical_dict.get("verdict") + summary = canonical_dict.get("summary") + if isinstance(verdict, str): + return verdict, summary if isinstance(summary, str) else None + status = result.get("status") + error = result.get("error") + return ( + status if isinstance(status, str) else "unknown", + error if isinstance(error, str) else None, + ) + + +def _render_results(results: list[dict[str, Any]]) -> None: + if not results: + typer.echo("No QA results found.") + return + for result in results: + verdict, summary = _result_verdict(result) + subject_id = result.get("subject_id", "-") + agent = result.get("agent_name") or result.get("qa_agent_id") or "-" + stale = " stale" if result.get("stale") is True else "" + line = f"{subject_id}\t{agent}\t{verdict}{stale}" + typer.echo(f"{line}\t{summary}" if summary else line) + + +def _matching_run_results( + raw_results: object, + *, + analysis_trace_ids: set[str], +) -> list[dict[str, Any]]: + results = _dict_list(raw_results, label="QA run history") + return [ + result for result in results if str(result.get("analysis_trace_id")) in analysis_trace_ids + ] + + +def _all_terminal(results: list[dict[str, Any]], analysis_trace_ids: set[str]) -> bool: + statuses = {str(result.get("analysis_trace_id")): result.get("status") for result in results} + return all(statuses.get(trace_id) in _TERMINAL_STATUSES for trace_id in analysis_trace_ids) + + +def _result_exit_code(results: list[dict[str, Any]]) -> int: + if any(result.get("status") == "error" for result in results): + return 3 + verdicts = [_result_verdict(result)[0] for result in results] + if any(verdict in {"failed", "unknown"} for verdict in verdicts): + return 1 + if any(verdict != "passed" for verdict in verdicts): + return 3 + return 0 + + +@qa_app.command("agents") +def list_agents( + subject_type: str = typer.Option( + "environment", + "--subject-type", + help="Resource scope: environment or taskset.", + ), + json_output: bool = typer.Option(False, "--json", help="Output the machine-readable response."), + limit: int = typer.Option(50, "--limit", min=1, max=500, help="Maximum agents to return."), + offset: int = typer.Option(0, "--offset", min=0, help="Number of agents to skip."), +) -> None: + """List QA agents available for a resource type.""" + platform = _platform() + normalized_type = _subject_type(subject_type) + try: + response = platform.get( + "/qa-agents", + params={"subject_type": normalized_type, "limit": limit, "offset": offset}, + ) + except HudRequestError as exc: + _request_error(str(exc)) + if not isinstance(response, dict) or not isinstance(response.get("items"), list): + typer.echo("Platform returned an invalid QA agent list.", err=True) + raise typer.Exit(3) + if json_output: + _print_json(response) + return + agents = _dict_list(response["items"], label="QA agent list") + if not agents: + typer.echo(f"No {normalized_type} QA agents found.") + return + for agent in agents: + typer.echo( + f"{agent.get('id', '-')}\t{agent.get('name', '-')}\t" + f"{agent.get('subject_type', '-')}\t{agent.get('model_name') or '-'}" + ) + + +@qa_app.command("run") +def run_agent( + agent_id: str = typer.Argument(..., help="QA agent UUID."), + subject_ids: list[str] = typer.Argument( # noqa: B008 + ..., + help="One or more Environment or Taskset UUIDs.", + ), + overwrite: bool = typer.Option( + False, + "--overwrite", + help="Create a fresh attempt even when current evidence already exists.", + ), + wait: bool = typer.Option( + True, + "--wait/--no-wait", + help="Wait for every launched analysis to finish.", + ), + timeout: float = typer.Option( + 900, + "--timeout", + min=1, + help="Maximum seconds to wait for QA execution.", + ), + json_output: bool = typer.Option(False, "--json", help="Output machine-readable results."), +) -> None: + """Run one QA agent against Environment or Taskset subjects.""" + platform = _platform() + try: + raw_runs = platform.post( + f"/qa-agents/{agent_id}/run-resources", + json={"subject_ids": subject_ids, "overwrite": overwrite}, + ) + except HudRequestError as exc: + _request_error(str(exc)) + runs = _dict_list(raw_runs, label="QA launch response") + if not runs: + if json_output: + _print_json([]) + else: + typer.echo("No new QA runs were created; current evidence was reused.") + return + if not wait: + if json_output: + _print_json(runs) + else: + _render_results(runs) + return + + analysis_trace_ids = { + str(run["analysis_trace_id"]) + for run in runs + if isinstance(run.get("analysis_trace_id"), str) + } + if len(analysis_trace_ids) != len(runs): + typer.echo("Platform returned QA runs without analysis trace IDs.", err=True) + raise typer.Exit(3) + + deadline = time.monotonic() + timeout + results: list[dict[str, Any]] = [] + while time.monotonic() < deadline: + try: + raw_results = platform.get( + f"/qa-agents/{agent_id}/runs", + params={"limit": 100}, + ) + except HudRequestError as exc: + _request_error(str(exc)) + results = _matching_run_results( + raw_results, + analysis_trace_ids=analysis_trace_ids, + ) + if _all_terminal(results, analysis_trace_ids): + break + time.sleep(_POLL_INTERVAL_SECONDS) + else: + if json_output: + _print_json(results) + else: + _render_results(results) + typer.echo(f"Timed out after {timeout:g}s waiting for QA runs.", err=True) + raise typer.Exit(3) + + if json_output: + _print_json(results) + else: + _render_results(results) + exit_code = _result_exit_code(results) + if exit_code: + raise typer.Exit(exit_code) + + +@qa_app.command("results") +def list_results( + subject_type: str = typer.Argument(..., help="Resource scope: environment or taskset."), + subject_ids: list[str] = typer.Argument( # noqa: B008 + ..., + help="One or more Environment or Taskset UUIDs.", + ), + json_output: bool = typer.Option(False, "--json", help="Output machine-readable results."), +) -> None: + """Inspect QA results attached to Environment or Taskset subjects.""" + platform = _platform() + normalized_type = _subject_type(subject_type) + try: + raw_results = platform.get( + "/qa-agents/results/resources", + params={"subject_type": normalized_type, "subject_ids": subject_ids}, + ) + except HudRequestError as exc: + _request_error(str(exc)) + results = _dict_list(raw_results, label="QA results") + if json_output: + _print_json(results) + else: + _render_results(results) diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py new file mode 100644 index 000000000..498a5eb30 --- /dev/null +++ b/hud/cli/tests/test_qa.py @@ -0,0 +1,244 @@ +"""CLI behavior for resource-scoped platform QA agents.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from hud.cli import app +from hud.utils.exceptions import HudRequestError + +runner = CliRunner() + +_AGENT_ID = "00000000-0000-4000-a000-000000000001" +_SUBJECT_ID = "00000000-0000-4000-a000-000000000002" +_TRACE_ID = "00000000-0000-4000-a000-000000000003" + + +def _agent() -> dict[str, object]: + return { + "id": _AGENT_ID, + "name": "Benchmark Coverage", + "subject_type": "taskset", + "scenario_name": "trace-explorer:taskset_benchmark_coverage", + "model_name": "claude-sonnet", + "public": False, + } + + +def _run(status: str = "queued") -> dict[str, object]: + return { + "id": "00000000-0000-4000-a000-000000000004", + "qa_agent_id": _AGENT_ID, + "subject_type": "taskset", + "subject_id": _SUBJECT_ID, + "analysis_trace_id": _TRACE_ID, + "status": status, + "attempt": 1, + } + + +def _result(verdict: str = "passed") -> dict[str, object]: + return { + "qa_agent_id": _AGENT_ID, + "subject_type": "taskset", + "subject_id": _SUBJECT_ID, + "agent_name": "Benchmark Coverage", + "analysis_trace_id": _TRACE_ID, + "status": "completed", + "canonical_result": { + "schema_version": "qa_agent_result.v1", + "verdict": verdict, + "summary": "Coverage is sufficient." if verdict == "passed" else "A gap was found.", + "findings": [], + "metadata": {}, + }, + "error": None, + "stale": False, + "attempt": 1, + } + + +def test_qa_agents_lists_resource_agents() -> None: + """Agent discovery forwards subject scope and renders stable identifiers.""" + platform = MagicMock() + platform.get.return_value = { + "items": [_agent()], + "total": 1, + "limit": 50, + "offset": 0, + } + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "agents", "--subject-type", "taskset"]) + + assert result.exit_code == 0 + assert "Benchmark Coverage" in result.output + assert _AGENT_ID in result.output + platform.get.assert_called_once_with( + "/qa-agents", + params={"subject_type": "taskset", "limit": 50, "offset": 0}, + ) + + +def test_qa_agents_json_preserves_platform_payload() -> None: + """Machine output retains pagination and agent fields without reshaping.""" + payload = {"items": [_agent()], "total": 1, "limit": 50, "offset": 0} + platform = MagicMock() + platform.get.return_value = payload + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "agents", "--subject-type", "taskset", "--json"], + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == payload + + +def test_qa_run_defaults_to_new_only_without_waiting() -> None: + """The safe default does not overwrite evidence or block for model execution.""" + platform = MagicMock() + platform.post.return_value = [_run()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--no-wait"], + ) + + assert result.exit_code == 0 + assert "queued" in result.output + platform.post.assert_called_once_with( + f"/qa-agents/{_AGENT_ID}/run-resources", + json={"subject_ids": [_SUBJECT_ID], "overwrite": False}, + ) + platform.get.assert_not_called() + + +def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: + """Waiting returns one for a completed quality failure, not an execution error.""" + platform = MagicMock() + platform.post.return_value = [_run()] + platform.get.side_effect = [ + [{**_run(), "status": "queued"}], + [_result("failed")], + ] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + patch("hud.cli.qa.time.sleep"), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--wait"], + ) + + assert result.exit_code == 1 + assert "failed" in result.output.lower() + assert "A gap was found." in result.output + assert platform.get.call_count == 2 + platform.get.assert_called_with(f"/qa-agents/{_AGENT_ID}/runs", params={"limit": 100}) + + +def test_qa_run_wait_timeout_is_execution_error() -> None: + """An exhausted local wait budget is not reported as a quality failure.""" + platform = MagicMock() + platform.post.return_value = [_run()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + patch("hud.cli.qa.time.monotonic", side_effect=[0, 2]), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--wait", "--timeout", "1"], + ) + + assert result.exit_code == 3 + assert "Timed out after 1s" in result.output + platform.get.assert_not_called() + + +def test_qa_run_rejects_missing_analysis_trace_contract() -> None: + """A malformed launch cannot enter a polling loop that never resolves.""" + platform = MagicMock() + run = _run() + del run["analysis_trace_id"] + platform.post.return_value = [run] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--wait"]) + + assert result.exit_code == 3 + assert "without analysis trace IDs" in result.output + platform.get.assert_not_called() + + +def test_qa_results_queries_repeated_subject_ids() -> None: + """Result inspection passes the canonical resource scope and identifiers.""" + platform = MagicMock() + platform.get.return_value = [_result()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "results", "taskset", _SUBJECT_ID, "--json"], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)[0]["canonical_result"]["verdict"] == "passed" + platform.get.assert_called_once_with( + "/qa-agents/results/resources", + params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, + ) + + +def test_qa_results_rejects_trace_scope_before_request() -> None: + """The resource CLI does not route trace subjects through the wrong API.""" + platform = MagicMock() + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "results", "trace", _SUBJECT_ID]) + + assert result.exit_code == 2 + assert "environment, taskset" in result.output + platform.get.assert_not_called() + + +def test_qa_request_failure_uses_request_error_exit() -> None: + """Authentication and platform failures stay distinct from quality verdicts.""" + platform = MagicMock() + platform.get.side_effect = HudRequestError("access denied", status_code=403) + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "agents", "--subject-type", "environment"]) + + assert result.exit_code == 2 + assert "access denied" in result.output diff --git a/hud/clients/client.py b/hud/clients/client.py index 78cf63ddc..2bbbbf2b4 100644 --- a/hud/clients/client.py +++ b/hud/clients/client.py @@ -159,9 +159,7 @@ async def hello(self, session_id: str | None = None) -> Manifest: version=env.get("version", "0.0.0"), ), bindings=bindings, - readiness=( - result["readiness"] if isinstance(result.get("readiness"), dict) else None - ), + readiness=(result["readiness"] if isinstance(result.get("readiness"), dict) else None), ) return self.manifest diff --git a/hud/utils/platform.py b/hud/utils/platform.py index 6184bbf4f..a87f1f155 100644 --- a/hud/utils/platform.py +++ b/hud/utils/platform.py @@ -46,7 +46,7 @@ def base_url(self) -> str: def url(self, path: str, params: dict[str, Any] | None = None) -> str: url = f"{self.base_url}{path}" if params: - url += "?" + urlencode(params) + url += "?" + urlencode(params, doseq=True) return url def get(self, path: str, *, params: dict[str, Any] | None = None) -> Any: diff --git a/hud/utils/tests/test_platform.py b/hud/utils/tests/test_platform.py index b1356614d..0261f0428 100644 --- a/hud/utils/tests/test_platform.py +++ b/hud/utils/tests/test_platform.py @@ -20,6 +20,19 @@ def test_url_prefixes_version_segment_and_joins_params() -> None: ) +def test_url_encodes_repeated_query_parameters() -> None: + """List-valued FastAPI query parameters are emitted as repeated keys.""" + platform = PlatformClient("https://api.example", "key") + + assert platform.url( + "/qa-agents/results/resources", + {"subject_type": "taskset", "subject_ids": ["first", "second"]}, + ) == ( + "https://api.example/v2/qa-agents/results/resources?" + "subject_type=taskset&subject_ids=first&subject_ids=second" + ) + + def test_get_and_post_route_through_shared_requests(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[dict[str, object]] = [] From f47cc80d9ddfa983ac08957305d550cef34f61fe Mon Sep 17 00:00:00 2001 From: CK0607 Date: Sun, 2 Aug 2026 16:23:41 +0530 Subject: [PATCH 4/7] fix(cli): scope QA polling and secret detection Poll launched resource subjects directly so concurrent history cannot hide runs, and normalize acronym-style readiness keys before applying the secret denylist. Co-authored-by: Cursor --- hud/cli/qa.py | 19 +++++++++++++++++-- hud/cli/tests/test_qa.py | 5 ++++- hud/environment/env.py | 12 ++++++++++-- hud/environment/tests/test_server.py | 8 ++++++-- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/hud/cli/qa.py b/hud/cli/qa.py index 210acdbcd..170031e5c 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -207,14 +207,29 @@ def run_agent( if len(analysis_trace_ids) != len(runs): typer.echo("Platform returned QA runs without analysis trace IDs.", err=True) raise typer.Exit(3) + launched_subject_ids = [ + str(run["subject_id"]) for run in runs if isinstance(run.get("subject_id"), str) + ] + launched_subject_types = { + str(run["subject_type"]) + for run in runs + if run.get("subject_type") in _RESOURCE_SUBJECT_TYPES + } + if len(launched_subject_ids) != len(runs) or len(launched_subject_types) != 1: + typer.echo("Platform returned QA runs without a consistent resource scope.", err=True) + raise typer.Exit(3) + launched_subject_type = launched_subject_types.pop() deadline = time.monotonic() + timeout results: list[dict[str, Any]] = [] while time.monotonic() < deadline: try: raw_results = platform.get( - f"/qa-agents/{agent_id}/runs", - params={"limit": 100}, + "/qa-agents/results/resources", + params={ + "subject_type": launched_subject_type, + "subject_ids": launched_subject_ids, + }, ) except HudRequestError as exc: _request_error(str(exc)) diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py index 498a5eb30..b794d1a65 100644 --- a/hud/cli/tests/test_qa.py +++ b/hud/cli/tests/test_qa.py @@ -151,7 +151,10 @@ def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: assert "failed" in result.output.lower() assert "A gap was found." in result.output assert platform.get.call_count == 2 - platform.get.assert_called_with(f"/qa-agents/{_AGENT_ID}/runs", params={"limit": 100}) + platform.get.assert_called_with( + "/qa-agents/results/resources", + params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, + ) def test_qa_run_wait_timeout_is_execution_error() -> None: diff --git a/hud/environment/env.py b/hud/environment/env.py index 571ed4c8a..245c58efd 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -63,6 +63,7 @@ class Answer(BaseModel, Generic[T]): "private_key", "refresh_token", "secret", + "secret_key", "secrets", "token", } @@ -77,15 +78,22 @@ class Answer(BaseModel, Generic[T]): "_private_key", "_secret", "_secret_access_key", + "_secret_key", "_token", ) +def _normalize_readiness_key(key: str) -> str: + """Normalize snake, kebab, camel, and acronym-prefixed credential keys.""" + value = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", key) + value = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", value) + return re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") + + def _readiness_has_secret_key(value: Any) -> bool: if isinstance(value, dict): for key, item in cast("dict[str, Any]", value).items(): - snake = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", key) - normalized = re.sub(r"[^a-z0-9]+", "_", snake.lower()).strip("_") + normalized = _normalize_readiness_key(key) if ( normalized in _READINESS_SECRET_KEYS or normalized.endswith(_READINESS_SECRET_SUFFIXES) diff --git a/hud/environment/tests/test_server.py b/hud/environment/tests/test_server.py index 993d7ce7d..4578a0dd9 100644 --- a/hud/environment/tests/test_server.py +++ b/hud/environment/tests/test_server.py @@ -46,7 +46,11 @@ async def test_hello_publishes_validated_readiness_declaration() -> None: assert client.manifest.readiness == readiness -def test_readiness_declaration_rejects_secret_args() -> None: +@pytest.mark.parametrize( + "secret_key", + ["apiKey", "APIKey", "MCPConfig", "secret_key"], +) +def test_readiness_declaration_rejects_secret_args(secret_key: str) -> None: """Environment authors cannot publish credentials as readiness arguments.""" with pytest.raises(ValidationError, match="must not contain"): Environment( @@ -55,7 +59,7 @@ def test_readiness_declaration_rejects_secret_args() -> None: "schema_version": "hud.environment-readiness.v0", "probe": { "scenario": "probe", - "args": {"nested": {"apiKey": "secret"}}, + "args": {"nested": {secret_key: "secret"}}, }, "reset": {"strategy": "reprovision"}, "budgets": { From c107789b971ef95e50f93da550cec30424770e72 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Sun, 2 Aug 2026 16:30:55 +0530 Subject: [PATCH 5/7] fix(cli): preserve reused QA verdicts Wait on every requested subject and include current reused evidence in output and exit-code evaluation, while pinning newly launched subjects to their exact analysis traces. Co-authored-by: Cursor --- hud/cli/qa.py | 74 ++++++++++++++++++++++++---------------- hud/cli/tests/test_qa.py | 67 +++++++++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 34 deletions(-) diff --git a/hud/cli/qa.py b/hud/cli/qa.py index 170031e5c..c213879f7 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -86,20 +86,33 @@ def _render_results(results: list[dict[str, Any]]) -> None: typer.echo(f"{line}\t{summary}" if summary else line) -def _matching_run_results( +def _matching_subject_results( raw_results: object, *, - analysis_trace_ids: set[str], + agent_id: str, + subject_ids: list[str], + launched_trace_ids: dict[str, str], ) -> list[dict[str, Any]]: - results = _dict_list(raw_results, label="QA run history") - return [ - result for result in results if str(result.get("analysis_trace_id")) in analysis_trace_ids - ] + results = _dict_list(raw_results, label="QA results") + expected_subject_ids = set(subject_ids) + matched: dict[str, dict[str, Any]] = {} + for result in results: + subject_id = str(result.get("subject_id")) + if str(result.get("qa_agent_id")) != agent_id or subject_id not in expected_subject_ids: + continue + launched_trace_id = launched_trace_ids.get(subject_id) + if ( + launched_trace_id is not None + and str(result.get("analysis_trace_id")) != launched_trace_id + ): + continue + matched[subject_id] = result + return [matched[subject_id] for subject_id in subject_ids if subject_id in matched] -def _all_terminal(results: list[dict[str, Any]], analysis_trace_ids: set[str]) -> bool: - statuses = {str(result.get("analysis_trace_id")): result.get("status") for result in results} - return all(statuses.get(trace_id) in _TERMINAL_STATUSES for trace_id in analysis_trace_ids) +def _all_terminal(results: list[dict[str, Any]], subject_ids: list[str]) -> bool: + statuses = {str(result.get("subject_id")): result.get("status") for result in results} + return all(statuses.get(subject_id) in _TERMINAL_STATUSES for subject_id in subject_ids) def _result_exit_code(results: list[dict[str, Any]]) -> int: @@ -179,6 +192,14 @@ def run_agent( """Run one QA agent against Environment or Taskset subjects.""" platform = _platform() try: + raw_agent = platform.get(f"/qa-agents/{agent_id}") + if ( + not isinstance(raw_agent, dict) + or raw_agent.get("subject_type") not in _RESOURCE_SUBJECT_TYPES + ): + typer.echo("Platform returned an invalid resource QA agent.", err=True) + raise typer.Exit(3) + agent_subject_type = str(raw_agent["subject_type"]) raw_runs = platform.post( f"/qa-agents/{agent_id}/run-resources", json={"subject_ids": subject_ids, "overwrite": overwrite}, @@ -186,7 +207,7 @@ def run_agent( except HudRequestError as exc: _request_error(str(exc)) runs = _dict_list(raw_runs, label="QA launch response") - if not runs: + if not wait and not runs: if json_output: _print_json([]) else: @@ -199,26 +220,17 @@ def run_agent( _render_results(runs) return - analysis_trace_ids = { - str(run["analysis_trace_id"]) + launched_trace_ids = { + str(run["subject_id"]): str(run["analysis_trace_id"]) for run in runs - if isinstance(run.get("analysis_trace_id"), str) + if isinstance(run.get("subject_id"), str) and isinstance(run.get("analysis_trace_id"), str) } - if len(analysis_trace_ids) != len(runs): + if len(launched_trace_ids) != len(runs): typer.echo("Platform returned QA runs without analysis trace IDs.", err=True) raise typer.Exit(3) - launched_subject_ids = [ - str(run["subject_id"]) for run in runs if isinstance(run.get("subject_id"), str) - ] - launched_subject_types = { - str(run["subject_type"]) - for run in runs - if run.get("subject_type") in _RESOURCE_SUBJECT_TYPES - } - if len(launched_subject_ids) != len(runs) or len(launched_subject_types) != 1: - typer.echo("Platform returned QA runs without a consistent resource scope.", err=True) + if not set(launched_trace_ids).issubset(subject_ids): + typer.echo("Platform returned QA runs for unexpected resources.", err=True) raise typer.Exit(3) - launched_subject_type = launched_subject_types.pop() deadline = time.monotonic() + timeout results: list[dict[str, Any]] = [] @@ -227,17 +239,19 @@ def run_agent( raw_results = platform.get( "/qa-agents/results/resources", params={ - "subject_type": launched_subject_type, - "subject_ids": launched_subject_ids, + "subject_type": agent_subject_type, + "subject_ids": subject_ids, }, ) except HudRequestError as exc: _request_error(str(exc)) - results = _matching_run_results( + results = _matching_subject_results( raw_results, - analysis_trace_ids=analysis_trace_ids, + agent_id=agent_id, + subject_ids=subject_ids, + launched_trace_ids=launched_trace_ids, ) - if _all_terminal(results, analysis_trace_ids): + if _all_terminal(results, subject_ids): break time.sleep(_POLL_INTERVAL_SECONDS) else: diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py index b794d1a65..7d03a577c 100644 --- a/hud/cli/tests/test_qa.py +++ b/hud/cli/tests/test_qa.py @@ -14,6 +14,7 @@ _AGENT_ID = "00000000-0000-4000-a000-000000000001" _SUBJECT_ID = "00000000-0000-4000-a000-000000000002" +_SECOND_SUBJECT_ID = "00000000-0000-4000-a000-000000000005" _TRACE_ID = "00000000-0000-4000-a000-000000000003" @@ -108,6 +109,7 @@ def test_qa_agents_json_preserves_platform_payload() -> None: def test_qa_run_defaults_to_new_only_without_waiting() -> None: """The safe default does not overwrite evidence or block for model execution.""" platform = MagicMock() + platform.get.return_value = _agent() platform.post.return_value = [_run()] with ( @@ -125,7 +127,7 @@ def test_qa_run_defaults_to_new_only_without_waiting() -> None: f"/qa-agents/{_AGENT_ID}/run-resources", json={"subject_ids": [_SUBJECT_ID], "overwrite": False}, ) - platform.get.assert_not_called() + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: @@ -133,6 +135,7 @@ def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: platform = MagicMock() platform.post.return_value = [_run()] platform.get.side_effect = [ + _agent(), [{**_run(), "status": "queued"}], [_result("failed")], ] @@ -150,16 +153,71 @@ def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: assert result.exit_code == 1 assert "failed" in result.output.lower() assert "A gap was found." in result.output - assert platform.get.call_count == 2 + assert platform.get.call_count == 3 + platform.get.assert_called_with( + "/qa-agents/results/resources", + params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, + ) + + +def test_qa_run_reused_failure_preserves_quality_exit() -> None: + """Run-new-only reuse still evaluates the stored result when waiting.""" + platform = MagicMock() + platform.post.return_value = [] + platform.get.side_effect = [_agent(), [_result("failed")]] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "run", _AGENT_ID, _SUBJECT_ID]) + + assert result.exit_code == 1 + assert "failed" in result.output.lower() + assert "A gap was found." in result.output platform.get.assert_called_with( "/qa-agents/results/resources", params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, ) +def test_qa_run_partial_reuse_waits_for_new_and_scores_all_subjects() -> None: + """A reused failure remains visible while another subject runs.""" + reused_failure = {**_result("failed"), "subject_id": _SECOND_SUBJECT_ID} + platform = MagicMock() + platform.post.return_value = [_run()] + platform.get.side_effect = [ + _agent(), + [{**_run(), "status": "queued"}, reused_failure], + [_result("passed"), reused_failure], + ] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + patch("hud.cli.qa.time.sleep"), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, _SECOND_SUBJECT_ID], + ) + + assert result.exit_code == 1 + assert _SUBJECT_ID in result.output + assert _SECOND_SUBJECT_ID in result.output + platform.get.assert_called_with( + "/qa-agents/results/resources", + params={ + "subject_type": "taskset", + "subject_ids": [_SUBJECT_ID, _SECOND_SUBJECT_ID], + }, + ) + + def test_qa_run_wait_timeout_is_execution_error() -> None: """An exhausted local wait budget is not reported as a quality failure.""" platform = MagicMock() + platform.get.return_value = _agent() platform.post.return_value = [_run()] with ( @@ -174,7 +232,7 @@ def test_qa_run_wait_timeout_is_execution_error() -> None: assert result.exit_code == 3 assert "Timed out after 1s" in result.output - platform.get.assert_not_called() + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") def test_qa_run_rejects_missing_analysis_trace_contract() -> None: @@ -182,6 +240,7 @@ def test_qa_run_rejects_missing_analysis_trace_contract() -> None: platform = MagicMock() run = _run() del run["analysis_trace_id"] + platform.get.return_value = _agent() platform.post.return_value = [run] with ( @@ -192,7 +251,7 @@ def test_qa_run_rejects_missing_analysis_trace_contract() -> None: assert result.exit_code == 3 assert "without analysis trace IDs" in result.output - platform.get.assert_not_called() + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") def test_qa_results_queries_repeated_subject_ids() -> None: From edd077fc96f55791d246b83b05222f0b7fc6f2a3 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Sun, 2 Aug 2026 16:38:10 +0530 Subject: [PATCH 6/7] fix(cli): canonicalize QA resource identifiers Match API-normalized UUIDs regardless of input casing and reject plural compound credential fields in readiness declarations. Co-authored-by: Cursor --- hud/cli/qa.py | 34 ++++++++++++++++++++-------- hud/cli/tests/test_qa.py | 19 ++++++++++++++++ hud/environment/env.py | 6 +++++ hud/environment/tests/test_server.py | 11 ++++++++- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/hud/cli/qa.py b/hud/cli/qa.py index c213879f7..1f5854705 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -57,6 +57,11 @@ def _print_json(value: object) -> None: typer.echo(json.dumps(value, indent=2, sort_keys=True, default=str)) +def _canonical_id(value: object) -> str: + """Match UUID-like identifiers independently of accepted hex casing.""" + return str(value).casefold() + + def _result_verdict(result: dict[str, Any]) -> tuple[str, str | None]: canonical = result.get("canonical_result") if isinstance(canonical, dict): @@ -94,25 +99,34 @@ def _matching_subject_results( launched_trace_ids: dict[str, str], ) -> list[dict[str, Any]]: results = _dict_list(raw_results, label="QA results") - expected_subject_ids = set(subject_ids) + expected_subject_ids = {_canonical_id(subject_id) for subject_id in subject_ids} matched: dict[str, dict[str, Any]] = {} for result in results: - subject_id = str(result.get("subject_id")) - if str(result.get("qa_agent_id")) != agent_id or subject_id not in expected_subject_ids: + subject_id = _canonical_id(result.get("subject_id")) + if ( + _canonical_id(result.get("qa_agent_id")) != _canonical_id(agent_id) + or subject_id not in expected_subject_ids + ): continue launched_trace_id = launched_trace_ids.get(subject_id) if ( launched_trace_id is not None - and str(result.get("analysis_trace_id")) != launched_trace_id + and _canonical_id(result.get("analysis_trace_id")) != launched_trace_id ): continue matched[subject_id] = result - return [matched[subject_id] for subject_id in subject_ids if subject_id in matched] + return [ + matched[canonical_id] + for subject_id in subject_ids + if (canonical_id := _canonical_id(subject_id)) in matched + ] def _all_terminal(results: list[dict[str, Any]], subject_ids: list[str]) -> bool: - statuses = {str(result.get("subject_id")): result.get("status") for result in results} - return all(statuses.get(subject_id) in _TERMINAL_STATUSES for subject_id in subject_ids) + statuses = {_canonical_id(result.get("subject_id")): result.get("status") for result in results} + return all( + statuses.get(_canonical_id(subject_id)) in _TERMINAL_STATUSES for subject_id in subject_ids + ) def _result_exit_code(results: list[dict[str, Any]]) -> int: @@ -221,14 +235,16 @@ def run_agent( return launched_trace_ids = { - str(run["subject_id"]): str(run["analysis_trace_id"]) + _canonical_id(run["subject_id"]): _canonical_id(run["analysis_trace_id"]) for run in runs if isinstance(run.get("subject_id"), str) and isinstance(run.get("analysis_trace_id"), str) } if len(launched_trace_ids) != len(runs): typer.echo("Platform returned QA runs without analysis trace IDs.", err=True) raise typer.Exit(3) - if not set(launched_trace_ids).issubset(subject_ids): + if not set(launched_trace_ids).issubset( + {_canonical_id(subject_id) for subject_id in subject_ids}, + ): typer.echo("Platform returned QA runs for unexpected resources.", err=True) raise typer.Exit(3) diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py index 7d03a577c..c9a4aff4d 100644 --- a/hud/cli/tests/test_qa.py +++ b/hud/cli/tests/test_qa.py @@ -181,6 +181,25 @@ def test_qa_run_reused_failure_preserves_quality_exit() -> None: ) +def test_qa_run_matches_canonical_results_for_uppercase_uuid_input() -> None: + """API-normalized UUID casing does not make a completed result disappear.""" + platform = MagicMock() + platform.post.return_value = [] + platform.get.side_effect = [_agent(), [_result("passed")]] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID.upper(), _SUBJECT_ID.upper()], + ) + + assert result.exit_code == 0 + assert "passed" in result.output.lower() + + def test_qa_run_partial_reuse_waits_for_new_and_scores_all_subjects() -> None: """A reused failure remains visible while another subject runs.""" reused_failure = {**_result("failed"), "subject_id": _SECOND_SUBJECT_ID} diff --git a/hud/environment/env.py b/hud/environment/env.py index 245c58efd..8f541da1d 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -71,15 +71,21 @@ class Answer(BaseModel, Generic[T]): _READINESS_SECRET_SUFFIXES = ( "_access_key", "_api_key", + "_api_keys", "_client_secret", "_credential", "_credentials", "_password", + "_passwords", "_private_key", + "_private_keys", "_secret", "_secret_access_key", "_secret_key", + "_secret_keys", + "_secrets", "_token", + "_tokens", ) diff --git a/hud/environment/tests/test_server.py b/hud/environment/tests/test_server.py index 4578a0dd9..305ff3baa 100644 --- a/hud/environment/tests/test_server.py +++ b/hud/environment/tests/test_server.py @@ -48,7 +48,16 @@ async def test_hello_publishes_validated_readiness_declaration() -> None: @pytest.mark.parametrize( "secret_key", - ["apiKey", "APIKey", "MCPConfig", "secret_key"], + [ + "apiKey", + "APIKey", + "MCPConfig", + "secret_key", + "db_secrets", + "api_tokens", + "db_passwords", + "signing_private_keys", + ], ) def test_readiness_declaration_rejects_secret_args(secret_key: str) -> None: """Environment authors cannot publish credentials as readiness arguments.""" From bb1e7d82898895d999c8e5d722204ff639a816b0 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Sun, 2 Aug 2026 16:45:59 +0530 Subject: [PATCH 7/7] fix(environment): allow token budget settings Keep credential-like token fields blocked while allowing well-known non-secret token counters and stop controls in readiness probe arguments. Co-authored-by: Cursor --- hud/environment/env.py | 31 ++++++++++++++++++++++++++-- hud/environment/tests/test_server.py | 23 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/hud/environment/env.py b/hud/environment/env.py index 8f541da1d..922e9cc31 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -84,8 +84,22 @@ class Answer(BaseModel, Generic[T]): "_secret_key", "_secret_keys", "_secrets", - "_token", - "_tokens", +) +_READINESS_NONSECRET_TOKEN_QUALIFIERS = frozenset( + { + "completion", + "context", + "estimated", + "input", + "max", + "num", + "output", + "prompt", + "remaining", + "stop", + "total", + "used", + } ) @@ -96,6 +110,15 @@ def _normalize_readiness_key(key: str) -> str: return re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") +def _is_nonsecret_token_setting(key: str) -> bool: + parts = key.split("_") + return ( + len(parts) > 1 + and parts[-1] in {"token", "tokens"} + and all(part in _READINESS_NONSECRET_TOKEN_QUALIFIERS for part in parts[:-1]) + ) + + def _readiness_has_secret_key(value: Any) -> bool: if isinstance(value, dict): for key, item in cast("dict[str, Any]", value).items(): @@ -103,6 +126,10 @@ def _readiness_has_secret_key(value: Any) -> bool: if ( normalized in _READINESS_SECRET_KEYS or normalized.endswith(_READINESS_SECRET_SUFFIXES) + or ( + normalized.endswith(("_token", "_tokens")) + and not _is_nonsecret_token_setting(normalized) + ) or _readiness_has_secret_key(item) ): return True diff --git a/hud/environment/tests/test_server.py b/hud/environment/tests/test_server.py index 305ff3baa..3d6b251c7 100644 --- a/hud/environment/tests/test_server.py +++ b/hud/environment/tests/test_server.py @@ -80,6 +80,29 @@ def test_readiness_declaration_rejects_secret_args(secret_key: str) -> None: ) +def test_readiness_declaration_allows_noncredential_token_settings() -> None: + """Token budgets and stop-token controls are not credentials.""" + env = Environment( + "safe", + readiness={ + "schema_version": "hud.environment-readiness.v0", + "probe": { + "scenario": "probe", + "args": {"max_tokens": 512, "stop_tokens": ["DONE"]}, + }, + "reset": {"strategy": "reprovision"}, + "budgets": { + "startup_timeout_s": 120, + "probe_timeout_s": 300, + "reset_timeout_s": 180, + }, + }, + ) + + assert env.readiness is not None + assert env.readiness.probe.args["max_tokens"] == 512 + + async def test_dict_grade_without_numeric_score_errors_loudly() -> None: env = Environment("badgrade")