From 82ef671b4ac81e7f72974e49e90aa827ff17e206 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:51:22 -0700 Subject: [PATCH 01/33] fix(environment): keep bwrap session argv on bubblewrap 0.4 options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--clearenv` arrived in bubblewrap 0.5.0, but debian bullseye — and every image built on it — ships 0.4.1. There, bwrap aborted on the unknown option before running anything, so every command in the session exited 1: the agent got a shell where `true` and `echo` failed, spent its budget diagnosing that, and the rollout still graded as a completed run scoring 0.0. Two of the ten terminal-bench 2.0 sample tasks are bullseye-based and lost exactly this way. `usable_bwrap` could not catch it: its probe creates namespaces but passes none of the options a session goes on to use, so an old bwrap looks healthy and then fails every command. Give the payload its environment with `env -i` instead, matching what the non-bwrap dropped-privilege path already does. Isolation is unchanged — the mounts and namespace flags all predate 0.4 — and the option set a session passes is now covered by a test rather than discovered on a user's image. --- hud/environment/tests/test_workspace.py | 60 +++++++++++++++++++++---- hud/environment/workspace.py | 20 +++++---- 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index ac18a50d8..367ccfceb 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools import os import sys import tempfile @@ -12,7 +13,7 @@ import pytest from hud.capabilities import SSHClient -from hud.environment.workspace import Workspace +from hud.environment.workspace import Mount, Workspace pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX workspace semantics") @@ -104,9 +105,15 @@ async def test_dropped_session_env_excludes_server_secrets( assert session_env["HOME"] == ws._guest_path +def _sandbox_env(argv: list[str]) -> dict[str, str]: + """The environment the sandboxed payload starts from (its ``env -i`` set).""" + assignments = argv[argv.index("-i") + 1 :] + return dict(item.split("=", 1) for item in itertools.takewhile(lambda a: "=" in a, assignments)) + + def test_bwrap_drops_host_env_when_walled(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """The bwrap path must not re-inject host secrets via --setenv, while - per-call env overrides still reach the sandbox.""" + """The bwrap path must not re-inject host secrets, while per-call env + overrides still reach the sandbox.""" monkeypatch.setenv("HUD_API_KEY", "super-secret") _wall(monkeypatch) @@ -114,10 +121,11 @@ def test_bwrap_drops_host_env_when_walled(tmp_path: Path, monkeypatch: pytest.Mo monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") argv = ws.shell_argv("echo hi", env={"PER_CALL": "1"}) - setenv_keys = {argv[i + 1] for i, tok in enumerate(argv) if tok == "--setenv"} - assert "HUD_API_KEY" not in setenv_keys - assert "CUSTOM" in setenv_keys and "PATH" in setenv_keys - assert "PER_CALL" in setenv_keys + sandbox_env = _sandbox_env(argv) + assert "HUD_API_KEY" not in sandbox_env + assert sandbox_env["CUSTOM"] == "1" + assert "PATH" in sandbox_env + assert sandbox_env["PER_CALL"] == "1" def test_bwrap_inherits_host_env_when_not_walled( @@ -127,8 +135,42 @@ def test_bwrap_inherits_host_env_when_not_walled( ws = Workspace(tmp_path / "root") monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") argv = ws.bwrap_argv(["bash", "-lc", "true"]) - setenv_keys = {argv[i + 1] for i, tok in enumerate(argv) if tok == "--setenv"} - assert "HUD_SENTINEL" in setenv_keys + assert _sandbox_env(argv)["HUD_SENTINEL"] == "visible" + + +#: Options bubblewrap gained after 0.4, the newest release on distros still in +#: use (debian bullseye ships 0.4.1). One of these in a session's argv aborts +#: every command on such a host with "Unknown option", which grades as a +#: legitimate zero rather than a broken environment. +_POST_0_4_BWRAP_OPTIONS = frozenset( + { + "--clearenv", # 0.5.0 + "--assert-userns-disabled", # 0.5.0 + "--overlay", # 0.8.0 + "--tmp-overlay", # 0.8.0 + "--ro-overlay", # 0.8.0 + "--overlay-src", # 0.8.0 + "--size", # 0.9.0 + "--chmod", # 0.9.0 + } +) + + +def test_session_argv_runs_on_bubblewrap_0_4( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Sessions must not pass an option an old-but-usable bwrap will reject.""" + ws = Workspace( + tmp_path / "root", + shell_uid=1000, + env={"CUSTOM": "1"}, + mounts=(Mount("tmpfs", dst="/tests"),), + ) + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + _wall(monkeypatch) + + for argv in (ws.shell_argv("echo hi"), ws.shell_argv(), ws.bwrap_argv(["true"])): + assert not _POST_0_4_BWRAP_OPTIONS.intersection(argv) def test_shell_uid_wraps_sessions_in_setpriv( diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 48c4f5a8e..36770ddcb 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -50,10 +50,12 @@ def usable_bwrap() -> str | None: if _bwrap_usable is None: try: probe = subprocess.run( - # Mirrors a real session's namespace setup — mounting proc is + # Mirrors a real session's *namespace* setup — mounting proc is # what an unprivileged container blocks first. The whole root # is bound so the probed binary keeps its loader; a narrower - # mount set fails for the wrong reason. + # mount set fails for the wrong reason. It does not prove the + # installed bwrap parses every option a session passes, so + # bwrap_argv stays within what bubblewrap 0.4 understands. [ path, "--unshare-user-try", @@ -474,9 +476,12 @@ def bwrap_argv( ) -> list[str]: """Argv that runs ``command`` inside bwrap. Raises if bwrap unavailable. - bwrap ``--clearenv`` then re-injects ``full_env`` via ``--setenv``, so - with ``inherit_host_env=False`` the host environment (server secrets) - is left out and only ``self.env`` + ``env`` reach the sandbox. + The payload runs under ``env -i``, so it starts from exactly + ``full_env``: with ``inherit_host_env=False`` the host environment + (server secrets) is left out and only ``self.env`` + ``env`` reach the + sandbox. Every option here is one bubblewrap 0.4 understands — + ``--clearenv`` (0.5+) would abort each session on an older bwrap that + :func:`usable_bwrap` cannot tell apart from a current one. """ if self._bwrap is None: raise RuntimeError("bwrap not available on this host") @@ -500,10 +505,9 @@ def bwrap_argv( for m in self.mounts: argv.extend(m.to_bwrap_args()) argv.extend(["--chdir", target_cwd]) - argv.append("--clearenv") - for k, v in full_env.items(): - argv.extend(["--setenv", k, v]) argv.append("--") + env_bin = shutil.which("env") or "/usr/bin/env" + argv.extend([env_bin, "-i", *(f"{k}={v}" for k, v in full_env.items())]) if isinstance(command, str): argv.extend(["bash", "-lc", command]) else: From 82472b78f25ce71683b31ce35e601142aab27bbc Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:51:50 -0700 Subject: [PATCH 02/33] fix(integrations): keep HUD's own state out of the graded filesystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapted layer installed uv and its managed interpreter at uv's defaults, which are relative to the invoking user's home — inside the very filesystem the task grades. The graded party then found a HUD runtime its task never declared, plus interpreter shims pointing into /hud, which the workspace masks and which therefore resolve to nothing. `hud`'s version check compounded it: no user to prompt, a cache that is never warm because the container is fresh each rollout, so it called PyPI on the rollout's critical path and wrote its state into the graded tree. Pin every uv directory under /hud, disable the cache nothing reads after the build, and turn the version check off in the image. /hud is the one path the workspace already masks, so containment is what makes that single mask sufficient instead of something to clean up afterwards. --- integrations/harbor/_adapt.py | 24 +++++++++++++++++---- integrations/harbor/tests/test_contract.py | 25 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index daa75870f..72d176757 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -100,7 +100,21 @@ # Needs network and one of: uv / curl / wget / pip (+ apt-get/apk for bare # images with no downloader). set -eu -export PATH="$HOME/.local/bin:$PATH" UV_INSTALL_DIR="$HOME/.local/bin" +# Everything HUD installs goes under /hud — the uv binary, its managed +# interpreter, that interpreter's shims — because /hud is the one path the +# workspace masks from the graded party. Left at uv's defaults this state lands +# in the invoking user's home, which is *inside* the task's filesystem: the +# agent then finds a HUD runtime the task never declared, and shims pointing +# into the mask that resolve to nothing. Containing it is what makes the single +# /hud mask sufficient, rather than something to be cleaned up after the fact. +# The cache is the same argument, minus the keeping: nothing reads it after this +# script, so it is never written. +export UV_INSTALL_DIR=/hud/bin \\ + UV_PYTHON_INSTALL_DIR=/hud/python \\ + UV_PYTHON_BIN_DIR=/hud/bin \\ + UV_NO_CACHE=1 \\ + XDG_CONFIG_HOME=/hud/config +export PATH="/hud/bin:$PATH" command -v uv >/dev/null 2>&1 || { { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \\ || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \\ @@ -118,9 +132,6 @@ || { apt-get update -qq && apt-get install -y -qq bubblewrap; } \ || apk add --no-cache bubblewrap \ || echo "warning: bubblewrap unavailable; tasks declaring isolation will refuse to serve" -# The interpreter must live under /hud: uv's default install dir is the -# invoking user's home (root here), unreachable once the image's USER applies. -export UV_PYTHON_INSTALL_DIR=/hud/python uv python install __PYTHON__ uv venv /hud/venv --python __PYTHON__ uv pip install --python /hud/venv/bin/python __HUD_REQUIREMENT__ @@ -139,6 +150,11 @@ # build context rather than the index — which also means the code serving the # image is the revision that adapted it, not whatever the index resolves to. COPY _hud_harbor /hud/venv/lib/python__PYTHON__/site-packages/harbor +# The CLI's update check has no user to prompt here, and its cache lives in the +# invoking user's home — inside the task's filesystem. A container is fresh +# every rollout, so the cache is never warm: left on, it calls PyPI on the +# rollout's critical path and leaves HUD state in the graded filesystem. +ENV HUD_SKIP_VERSION_CHECK=1 EXPOSE 8765 __DECLARED__ENTRYPOINT [] CMD ["/hud/venv/bin/hud", "serve", \ diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index bb9c045a8..cd17f5145 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -9,6 +9,7 @@ from __future__ import annotations import os +import re from typing import TYPE_CHECKING import pytest @@ -69,6 +70,30 @@ async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None: assert (task_dir / "tests" / "test.sh").is_file() +async def test_the_layer_keeps_its_own_state_under_the_mask(tmp_path) -> None: + # /hud is the path the workspace masks from the graded party, so everything + # the layer installs belongs there: uv's binary, its managed interpreter and + # that interpreter's shims, its cache and config. At uv's defaults these + # land in the invoking user's home — inside the task's own filesystem, where + # the agent finds a HUD runtime the task never declared, and shims into the + # mask that resolve to nothing. + _write_harbor_task(tmp_path, "task-a") + await harbor.adapt(tmp_path, build=False) + (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) + script = (context / "_hud" / "install.sh").read_text(encoding="utf-8") + + directories = dict(re.findall(r"\b(UV_\w*(?:DIR|HOME)|XDG_CONFIG_HOME)=(\S+)", script)) + assert directories, "the layer pins no uv directories" + assert all(path.startswith("/hud/") for path in directories.values()), directories + # Nothing is written relative to the image's home, so there is no scattered + # state to clean up afterwards. + assert "$HOME" not in script + # The CLI's update check writes into that home too, and calls PyPI on every + # rollout since a fresh container never has a warm cache. + dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") + assert "ENV HUD_SKIP_VERSION_CHECK=1" in dockerfile + + def test_adapt_images_stamp_rows_when_the_caller_passes_them(tmp_path) -> None: # The mapping is a value the caller holds — nothing is written into the # dataset, so there is no cache to go stale. From a6ab9adc3dd3dd5d8e5d45949a9800e6e66fadea Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:51:59 -0700 Subject: [PATCH 03/33] fix(integrations): match Harbor's agent-phase environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways an adapted image differed from what Harbor gives the same task, both of which move the score without any model being involved. Harbor uploads /tests when it runs the verifier, so the agent phase never contains the assertions being graded; the adapted image laid that directory down before the agent ran. Mask it like the rest of the grading material and let _grade lay the baked copy down at the point Harbor would. Harbor also installs its agent *into* the container, provisioning python3, pip, git and curl (BaseInstalledAgent.SYSTEM_PACKAGES) before the agent phase. HUD's agent is external and installs nothing, so an adapted image handed the agent a barer machine than Harbor would — a harness difference scored as a model difference. Bake the same set at build time, presence-checked per tool so an image shipping its own interpreter keeps exactly that one. --- integrations/harbor/_adapt.py | 44 +++++++++++++++++++--- integrations/harbor/tests/test_contract.py | 44 ++++++++++++++++++++++ 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 72d176757..5a49046cc 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -135,6 +135,36 @@ uv python install __PYTHON__ uv venv /hud/venv --python __PYTHON__ uv pip install --python /hud/venv/bin/python __HUD_REQUIREMENT__ + +# ─── the agent's toolchain ─── +# Harbor installs the agent *into* the container, and that step provisions what +# the agent needs (BaseInstalledAgent.SYSTEM_PACKAGES: python3, pip, git, curl) +# before the agent phase. HUD's agent is external and installs nothing, so +# without this an adapted image hands the agent a barer machine than Harbor +# would for the same task — a difference in the harness, scored as if it were a +# difference in the model. +# +# Build time, not per rollout: baked into the content-addressed layer once and +# reused by every rollout. Only what the image actually lacks is installed, so +# an image shipping its own Python keeps exactly that Python. +apt_pkgs="" +apk_pkgs="" +for spec in \\ + "python3|python3 python3-venv|python3" \\ + "pip3|python3-pip|py3-pip" \\ + "git|git|git" \\ + "curl|curl ca-certificates|curl ca-certificates" +do + if command -v "${spec%%|*}" >/dev/null 2>&1; then continue; fi + rest=${spec#*|} + apt_pkgs="$apt_pkgs ${rest%%|*}" + apk_pkgs="$apk_pkgs ${rest##*|}" +done +if [ -n "$apt_pkgs" ]; then + { apt-get update -qq && apt-get install -y -qq $apt_pkgs; } \\ + || apk add --no-cache $apk_pkgs \\ + || echo "warning: could not provision the agent toolchain:$apt_pkgs" +fi """ _LAYER = """ @@ -405,11 +435,13 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E # bind — as system mounts they would be re-covered when the guest # path is ``/`` (an image with no WORKDIR). The graded party's # namespace must not contain the grading material or the verdict: - # the baked tests and serving venv (/hud) and the verifier's output + # the baked tests and serving venv (/hud), the verifier Harbor only + # uploads once the agent is done (/tests), and the verifier's output # dir are throwaway tmpfs here, while grading runs outside this # namespace and sees the real ones. mounts=( Mount("tmpfs", dst="/hud"), + Mount("tmpfs", dst=str(TESTS)), Mount("tmpfs", dst=str(VERIFIER_LOGS)), ), track_files=False if rooted_at_filesystem else None, @@ -435,11 +467,11 @@ def _register(env: Environment, task_dir: Path, workdir: Path) -> None: description=config.task.description or f"Harbor task {task_dir.name}", ) async def _run_harbor_task() -> AsyncGenerator[Any, Any]: - # Harbor's harness provides /tests during the agent phase, holding - # *this* task's verifier. One adapted image serves a whole group and - # may serve many rollouts, so the directory is laid down per rollout - # and emptied afterwards: no agent ever sees another task's tests. - _sync_tests(task_dir) + # Harbor uploads /tests when it runs the verifier, so the agent phase + # never contains the assertions it is graded on; :func:`_grade` lays + # the baked copy down at that same point. One adapted image serves a + # whole group and may serve many rollouts, so the directory is emptied + # afterwards: no agent ever sees another task's tests. try: answer = yield (task_dir / "instruction.md").read_text(encoding="utf-8") yield await _grade(task_dir, workdir, answer) diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index cd17f5145..6d2810882 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -10,6 +10,7 @@ import os import re +import subprocess from typing import TYPE_CHECKING import pytest @@ -94,6 +95,46 @@ async def test_the_layer_keeps_its_own_state_under_the_mask(tmp_path) -> None: assert "ENV HUD_SKIP_VERSION_CHECK=1" in dockerfile +async def test_the_agent_toolchain_installs_only_what_the_image_lacks(tmp_path) -> None: + # Harbor's agent install provisions python3/pip/git/curl into the task + # container before the agent phase; an adapted image bakes the same set at + # build time instead. An image shipping its own interpreter must keep it, + # so every tool is presence-checked rather than installed outright. + _write_harbor_task(tmp_path, "task-a") + await harbor.adapt(tmp_path, build=False) + (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) + script = (context / "_hud" / "install.sh").read_text(encoding="utf-8") + decision = script[script.index('apt_pkgs=""') : script.index("\ndone") + len("\ndone")] + + def queued(path: str) -> set[str]: + # Runs under ``set -eu`` as the image does: a tool already present must + # not abort the script mid-build. + result = subprocess.run( + ["/bin/sh", "-c", f'set -eu\nPATH={path}\n{decision}\nprintf "%s" "$apt_pkgs"'], + capture_output=True, + text=True, + check=True, + ) + return set(result.stdout.split()) + + assert queued("/nonexistent") == { + "python3", + "python3-venv", + "python3-pip", + "git", + "curl", + "ca-certificates", + } + + stub = tmp_path / "bin" + stub.mkdir() + for tool in ("python3", "pip3"): + (stub / tool).write_text("#!/bin/sh\n", encoding="utf-8") + (stub / tool).chmod(0o755) + + assert queued(str(stub)) == {"git", "curl", "ca-certificates"} + + def test_adapt_images_stamp_rows_when_the_caller_passes_them(tmp_path) -> None: # The mapping is a value the caller holds — nothing is written into the # dataset, so there is no cache to go stale. @@ -433,6 +474,9 @@ def record(*args, **kwargs): masked = [m.dst for m in workspace.mounts] assert "/hud" in masked assert "/logs/verifier" in masked + # Harbor uploads the verifier only once the agent is done, so the graded + # party never has the assertions it is graded on. + assert "/tests" in masked assert [m.dst for m in workspace._system_mounts] == ["/", "/proc", "/dev"] From 4c7b3eb107e26e00e54b395ec6535b2703a84f99 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:39:15 -0700 Subject: [PATCH 04/33] fix(environment): keep one sandbox per rollout, not one per command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every exec channel built its own bwrap sandbox, so each command ran in a fresh pid namespace that died with it. Files persisted — the root is a real directory — which made the workspace look stateful right up to the point an agent started something and used it in the next command. `nohup`, `setsid`, a listener: all gone, with nothing to say so. That makes a whole class of task unsolvable while scoring like a weak agent. Both qemu tasks in the terminal-bench 2.0 sample ("start the image in the background and leave it running") and configure-git-webserver graded 0.0 on it; the qemu pair passes now. Harbor has no such problem because its agent works in one persistent tmux session in the container. Hold one sandbox and have sessions join it with nsenter. The user namespace is joined first, which is what confers the privilege to join the rest in a container given no extra capability; the network namespace is joined only when the sandbox owns one, since a sandbox sharing the container's netns cannot rejoin it from inside bwrap's userns. The pid comes from bwrap's --info-fd, and readiness from the holder's own first line of output: bwrap names the child before that child finishes building its mount namespace, so the pid alone can point at a half-assembled root. A command that completes now keeps its process group, since `some-server &` is how an agent starts what it means to use next; the sandbox is the lifetime boundary instead, discarded at the rollout boundary so a reused container never hands the next agent the last one's daemons. Commands that time out, get abandoned, or run with no sandbox are still torn down as a group, having nothing else to bound them. --- hud/environment/tests/test_workspace.py | 79 ++++++++ hud/environment/workspace.py | 248 +++++++++++++++++++++--- integrations/harbor/_adapt.py | 13 +- 3 files changed, 308 insertions(+), 32 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 367ccfceb..4e624348f 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -2,11 +2,13 @@ from __future__ import annotations +import asyncio import itertools import os import sys import tempfile from pathlib import Path +from types import SimpleNamespace from typing import Any, cast import asyncssh @@ -173,6 +175,83 @@ def test_session_argv_runs_on_bubblewrap_0_4( assert not _POST_0_4_BWRAP_OPTIONS.intersection(argv) +def test_sessions_join_one_sandbox_rather_than_each_making_its_own( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two commands must land in the same namespaces, or a process the first + backgrounds is gone by the second.""" + ws = Workspace(tmp_path / "root", guest_path="/app", network=True) + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + + first = ws.enter_argv(4321, "start-a-server &") + second = ws.enter_argv(4321, "curl localhost") + + assert first[0].endswith("/nsenter") and second[0].endswith("/nsenter") + # Same target, so the same live namespaces — not a fresh sandbox per command. + assert first[first.index("--target") + 1] == "4321" + assert second[second.index("--target") + 1] == "4321" + assert "--pid" in first and "--mount" in first and "--user" in first + # The user namespace has to be joined first: it is what confers the + # privilege to join the others in a container given no extra capability. + assert first.index("--user") < min(first.index("--mount"), first.index("--pid")) + assert "--wd=/app" in first + + +@pytest.mark.asyncio +async def test_concurrent_sessions_share_one_sandbox( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An agent issues parallel tool calls; if each started its own sandbox, + what one backgrounds would be invisible to the next.""" + ws = Workspace(tmp_path / "root") + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + spawned = 0 + + async def fake_spawn() -> int: + nonlocal spawned + spawned += 1 + await asyncio.sleep(0.01) # the real spawn awaits bwrap's readiness + ws._sandbox = cast("Any", SimpleNamespace(returncode=None)) + ws._sandbox_init = 4000 + spawned + return ws._sandbox_init + + monkeypatch.setattr(ws, "_start_sandbox", fake_spawn) + + pids = await asyncio.gather(*(ws.sandbox_pid() for _ in range(4))) + + assert spawned == 1 + assert set(pids) == {4001} + + +def test_a_sharing_sandbox_is_not_rejoined_by_network_namespace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Joining bwrap's user namespace forfeits authority over the container's + netns, so asking for it fails the session outright; a severed sandbox owns + its netns and must be joined or the network comes back.""" + shared = Workspace(tmp_path / "shared", network=True) + severed = Workspace(tmp_path / "severed", network=False) + for ws in (shared, severed): + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + + assert "--net" not in shared.enter_argv(11, "true") + assert "--net" in severed.enter_argv(11, "true") + + +def test_the_sandbox_reports_readiness_before_sessions_join_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """bwrap names the child pid before that child has built its mount + namespace, so the pid alone is not proof the sandbox can run anything.""" + ws = Workspace(tmp_path / "root") + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + argv = ws.bwrap_argv(["sh", "-c", "echo ready"], info_fd=7) + + assert argv[argv.index("--info-fd") + 1] == "7" + # The signal comes from the payload, which bwrap runs only after setup. + assert argv[-1] == "echo ready" + + def test_shell_uid_wraps_sessions_in_setpriv( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 36770ddcb..c4c8cae14 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -4,6 +4,7 @@ import asyncio import contextlib +import json import logging import os import shutil @@ -143,6 +144,39 @@ def to_bwrap_args(self) -> list[str]: _DEFAULT_USER = "agent" +#: What the sandbox runs so it stays alive between sessions. It must outlast +#: every rollout without waking (a sandbox is discarded, never expired) and +#: come from the task's own image, since the serving venv is masked inside. +#: bwrap's reaper is pid 1 above it, so processes the agent orphans are reaped +#: rather than accumulating for the life of the sandbox. +#: +#: The line it prints first is the readiness signal, and it has to come from +#: in here: bwrap reports the child pid before that child has finished +#: building its mount namespace, so a session joining on the strength of the +#: pid alone can land in a root that is still half-assembled. The payload runs +#: only once setup is done, so its own output is the proof. +_SANDBOX_HOLDER = ["sh", "-c", "echo ready; exec sleep 2147483647"] +_SANDBOX_READY = b"ready\n" + + +def _env_argv(env: Mapping[str, str]) -> list[str]: + """``env -i`` and its assignments: an exact environment for what follows. + + Understood by every bubblewrap, unlike ``--clearenv``, which 0.4 lacks. + """ + env_bin = shutil.which("env") or "/usr/bin/env" + return [env_bin, "-i", *(f"{k}={v}" for k, v in env.items())] + + +def _payload_argv(command: str | list[str] | None, env: Mapping[str, str]) -> list[str]: + """The session itself: a login shell (or an exact argv) under ``env``.""" + argv = _env_argv(env) + if isinstance(command, str): + return [*argv, "bash", "-lc", command] + if command is None: + return [*argv, "bash", "-l"] + return argv + command + class Workspace: """Directory + bwrap-isolated SSH. @@ -242,6 +276,14 @@ def __init__( self._ft_server: asyncio.Server | None = None self._ft_host: str | None = None self._ft_port: int | None = None + # The sandbox sessions run in, spawned on first use and held until + # discard_sandbox(). Its namespaces are what makes a process the agent + # backgrounds outlive the command that started it. + self._sandbox: asyncio.subprocess.Process | None = None + self._sandbox_init: int | None = None + # Sessions start concurrently (an agent can issue parallel tool calls), + # and two that each started a sandbox would not share one. + self._sandbox_lock = asyncio.Lock() def _setpriv(self) -> str | None: """Absolute path to ``setpriv``, resolved via the *server's* PATH. @@ -365,6 +407,7 @@ async def stop(self) -> None: Credentials stay on disk; a later :meth:`start` re-binds (fresh port unless one was pinned) and reuses them. """ + await self.discard_sandbox() if self._ft_server is not None: self._ft_server.close() with contextlib.suppress(Exception): @@ -473,6 +516,7 @@ def bwrap_argv( cwd: str | None = None, env: Mapping[str, str] | None = None, inherit_host_env: bool = True, + info_fd: int | None = None, ) -> list[str]: """Argv that runs ``command`` inside bwrap. Raises if bwrap unavailable. @@ -499,6 +543,8 @@ def bwrap_argv( ] if not self.network: argv.append("--unshare-net") + if info_fd is not None: + argv.extend(["--info-fd", str(info_fd)]) for m in self._system_mounts: argv.extend(m.to_bwrap_args()) argv.extend(["--bind", str(self.root), self._guest_path]) @@ -506,14 +552,162 @@ def bwrap_argv( argv.extend(m.to_bwrap_args()) argv.extend(["--chdir", target_cwd]) argv.append("--") - env_bin = shutil.which("env") or "/usr/bin/env" - argv.extend([env_bin, "-i", *(f"{k}={v}" for k, v in full_env.items())]) - if isinstance(command, str): - argv.extend(["bash", "-lc", command]) - else: - argv.extend(command) + argv.extend(_payload_argv(command, full_env)) + return argv + + def enter_argv( + self, + pid: int, + command: str | None = None, + *, + cwd: str | None = None, + env: Mapping[str, str] | None = None, + ) -> list[str]: + """Argv that runs ``command`` inside the sandbox *pid* belongs to. + + The counterpart to :meth:`bwrap_argv`, which *creates* a sandbox: this + joins one that already exists, so successive commands share it. The + user namespace is joined first — that is what grants the privileges to + join the rest without any capability the container was not given. + + The network namespace is joined only when the sandbox has one of its + own, which is exactly when the workspace severed the network. A + sharing sandbox is already in this process's netns, and that netns + belongs to an outer user namespace: once joined to bwrap's, we hold no + authority there and rejoining fails outright. + """ + nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" + argv = [ + nsenter, + "--target", + str(pid), + "--user", + "--mount", + "--pid", + "--uts", + "--ipc", + *(() if self.network else ("--net",)), + f"--wd={cwd if cwd is not None else self._guest_path}", + "--", + ] + # Unlike the bwrap path, the drop goes *inside*: joining namespaces + # needs the privileges the dropped uid does not have. + argv.extend(self._drop_argv()) + argv.extend(_payload_argv(command, self._full_env(env))) return argv + def _full_env(self, env: Mapping[str, str] | None = None) -> dict[str, str]: + """The environment a session starts from. + + Dropped sessions get the minimal one built for the wall; otherwise the + serving process's environment carries through, as it always has. + """ + if self._drops_privileges(): + return {**(self._session_env() or {}), **(env or {})} + return {**os.environ, **self.env, **(env or {})} + + def _drop_argv(self) -> list[str]: + """The ``setpriv`` prefix that drops to ``shell_uid``, if it applies.""" + if not self._drops_privileges(): + return [] + setpriv = self._setpriv() + assert setpriv is not None # guaranteed by _drops_privileges + uid = str(self._shell_uid) + # --no-new-privs: without it a setuid binary (or passwordless sudo) + # inside the workspace would let the dropped shell regain root and + # read the secrets the wall protects. + return [setpriv, "--reuid", uid, "--regid", uid, "--clear-groups", "--no-new-privs", "--"] + + # ─── the sandbox sessions share ─────────────────────────────────────── + + async def sandbox_pid(self) -> int | None: + """The live sandbox's init pid, starting one if none is running. + + ``None`` where bwrap cannot sandbox: sessions then run directly, as + they always have, and nothing persists between them beyond the files. + """ + if self._bwrap is None: + return None + if (live := self._live_sandbox_pid()) is not None: + return live + async with self._sandbox_lock: + # Another session may have started it while this one waited. + if (live := self._live_sandbox_pid()) is not None: + return live + return await self._start_sandbox() + + def _live_sandbox_pid(self) -> int | None: + if self._sandbox is None or self._sandbox.returncode is not None: + return None + assert self._sandbox_init is not None + return self._sandbox_init + + async def _start_sandbox(self) -> int: + """Spawn the holder whose namespaces sessions join, and learn its pid. + + The pid comes from bwrap's ``--info-fd`` rather than by searching for + the holder: the pid that matters is the one *this* process can name, + and bwrap is the only party that knows which of its forks that is. + """ + read_fd, write_fd = os.pipe() + try: + os.set_inheritable(write_fd, True) + argv = self.bwrap_argv(_SANDBOX_HOLDER, info_fd=write_fd) + self._sandbox = await asyncio.create_subprocess_exec( + *argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + pass_fds=(write_fd,), + ) + os.close(write_fd) + write_fd = -1 + loop = asyncio.get_running_loop() + raw = await asyncio.wait_for(loop.run_in_executor(None, os.read, read_fd, 4096), 30.0) + finally: + os.close(read_fd) + if write_fd != -1: + os.close(write_fd) + if not raw: + raise RuntimeError(f"the sandbox holder did not start: {await self._sandbox_error()}") + pid = int(json.loads(raw)["child-pid"]) + assert self._sandbox is not None and self._sandbox.stdout is not None + try: + signal = await asyncio.wait_for(self._sandbox.stdout.readline(), 30.0) + except TimeoutError: + signal = b"" + if signal != _SANDBOX_READY: + reason = await self._sandbox_error() + await self.discard_sandbox() + raise RuntimeError(f"the sandbox never became ready: {reason}") + self._sandbox_init = pid + return pid + + async def _sandbox_error(self) -> str: + """Whatever the sandbox said on the way down, for a failure message.""" + if self._sandbox is None or self._sandbox.stderr is None: + return "no output" + with contextlib.suppress(Exception): + stderr = await asyncio.wait_for(self._sandbox.stderr.read(2048), 5.0) + return stderr.decode(errors="replace").strip() or "no output" + return "no output" + + async def discard_sandbox(self) -> None: + """Tear the sandbox down, killing everything still running in it. + + The rollout boundary: one adapted image serves many rollouts, and a + sandbox that outlives its own would hand the next agent the previous + one's daemons. Killing the holder collapses the pid namespace, so + every process the agent left behind goes with it — including ones it + detached. A later session starts a fresh sandbox. + """ + sandbox, self._sandbox, self._sandbox_init = self._sandbox, None, None + if sandbox is None or sandbox.returncode is not None: + return + sandbox.kill() + with contextlib.suppress(Exception): + await asyncio.wait_for(sandbox.wait(), 10.0) + def shell_argv( self, command: str | None = None, @@ -549,26 +743,8 @@ def shell_argv( # The session env (self.env + per-call overrides) is injected # only *after* the drop: vars like LD_PRELOAD in it must never # be in the environment of the root-run setpriv itself. - session = {**(self._session_env() or {}), **(env or {})} - env_bin = shutil.which("env") or "/usr/bin/env" - argv = [env_bin, "-i", *[f"{k}={v}" for k, v in session.items()], *argv] - setpriv = self._setpriv() - assert setpriv is not None # guaranteed by _drops_privileges - uid = str(self._shell_uid) - # --no-new-privs: without it a setuid binary (or passwordless - # sudo) inside the workspace would let the dropped shell regain - # root and read the secrets the wall protects. - argv = [ - setpriv, - "--reuid", - uid, - "--regid", - uid, - "--clear-groups", - "--no-new-privs", - "--", - *argv, - ] + argv = [*_env_argv(self._full_env(env)), *argv] + argv = [*self._drop_argv(), *argv] return argv # ─── ssh server internals ───────────────────────────────────────── @@ -641,7 +817,12 @@ def _session_env(self) -> dict[str, str] | None: return {**os.environ, **self.env} if self.env else None async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> None: - argv = self.shell_argv(process.command) + pid = await self.sandbox_pid() + argv = ( + self.shell_argv(process.command) + if pid is None + else self.enter_argv(pid, process.command) + ) if self._drops_privileges(): # The pre-drop processes (setpriv, bwrap) run as root; caller env # like an LD_PRELOAD in self.env must not load into them. The @@ -780,7 +961,18 @@ async def drain_output(reader: asyncio.StreamReader, output: bytearray) -> None: except TimeoutError: timed_out = True finally: - await sub.terminate() + # A command that ran to completion inside the sandbox keeps its + # process group: `some-server &` is how an agent starts something + # it means to use in the *next* command, and killing the group + # here would take it down with the shell that launched it. The + # sandbox is the lifetime boundary instead — discarding it at the + # end of the rollout collapses the pid namespace and everything + # left in it. Nothing bounds a command that timed out, was + # abandoned mid-flight, or ran with no sandbox at all, so those + # are still torn down as a group. + completed = wait_task.done() and not wait_task.cancelled() + if pid is None or timed_out or not completed: + await sub.terminate() stdin_task.cancel() wait_task.cancel() channel_closed_task.cancel() diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 5a49046cc..2721c30f0 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -48,7 +48,7 @@ from typing import Any from hud.environment import Environment, Mount -from hud.environment.workspace import usable_bwrap +from hud.environment.workspace import Workspace, usable_bwrap from hud.eval import DockerRuntime from hud.utils.docker import docker as _docker from hud.utils.process import ProcessGroup, create_process_group_exec @@ -423,7 +423,7 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E # its real paths — the sandbox is here to control the network namespace # and to keep the graded material (baked tests, the serving venv) out of # the graded party's reach, not to narrow the filesystem. - env.workspace( + workspace = env.workspace( workdir, guest_path=workdir.as_posix(), system_mounts=( @@ -455,11 +455,11 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E ) for task_dir in task_dirs: - _register(env, task_dir, workdir) + _register(env, task_dir, workdir, workspace) return env -def _register(env: Environment, task_dir: Path, workdir: Path) -> None: +def _register(env: Environment, task_dir: Path, workdir: Path, workspace: Workspace) -> None: config = TaskConfig.read(task_dir) @env.template( @@ -477,6 +477,11 @@ async def _run_harbor_task() -> AsyncGenerator[Any, Any]: yield await _grade(task_dir, workdir, answer) finally: _reset_dir(TESTS) + # Harbor's agent phase is one continuous session, so a service the + # agent starts is still running when the verifier looks for it — + # and is gone before the next rollout, which this container may + # well serve too. + await workspace.discard_sandbox() def _sync_tests(task_dir: Path) -> None: From 153152b4b00e7c39983225ff8558ae45be45312e Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:44:15 -0700 Subject: [PATCH 05/33] fix(environment): stream session output instead of holding it until exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Output was accumulated in the handler and written to the channel only once the process finished. An agent running a build or a test suite saw nothing until it ended, and a session that does not exit — a shell held open across calls — produced nothing at all, which foreclosed any harness tool built on a persistent session. Worse, a command stopped at the timeout reported only that the deadline had passed: the epilogue returned before writing what had been collected, so the output showing how far it got was discarded precisely when it was the only evidence available. Relay both streams as they arrive. The timeout message then says why the command stopped rather than standing in for its output, and a held-open shell reports each command as it runs — with cwd, exports and aliases persisting, since it is one shell — leaving what tools to build on that to the harness, which is where that choice belongs. --- hud/environment/tests/test_workspace.py | 43 +++++++++++++++++++++++++ hud/environment/workspace.py | 31 +++++++++++------- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 4e624348f..47b517109 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -7,6 +7,7 @@ import os import sys import tempfile +import time from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -15,6 +16,7 @@ import pytest from hud.capabilities import SSHClient +from hud.environment import workspace as workspace_mod from hud.environment.workspace import Mount, Workspace pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX workspace semantics") @@ -84,6 +86,47 @@ async def test_file_operations_use_the_exec_channel(tmp_path: Path) -> None: await ws.stop() +@pytest.mark.asyncio +async def test_output_arrives_while_the_command_is_still_running(tmp_path: Path) -> None: + """Held until exit, a long build tells the agent nothing while it runs and + a session that never exits says nothing at all.""" + ws = Workspace(tmp_path / "root") + await ws.start() + try: + async with await _connect(ws) as conn: + started = time.monotonic() + process = await conn.create_process("echo first; sleep 5; echo second") + first = await asyncio.wait_for(process.stdout.readline(), 10) + elapsed = time.monotonic() - started + process.channel.close() + finally: + await ws.stop() + + assert first.strip() == "first" + # Held until exit it would take the full sleep; the point is that it does not. + assert elapsed < 2.0, f"first line took {elapsed:.1f}s — output is not streaming" + + +@pytest.mark.asyncio +async def test_a_timed_out_command_keeps_what_it_printed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The output is the evidence of how far it got — reporting only that the + deadline passed throws that away.""" + monkeypatch.setattr(workspace_mod, "_COMMAND_TIMEOUT", 1.0) + ws = Workspace(tmp_path / "root") + await ws.start() + try: + async with await _connect(ws) as conn: + result = await conn.run("echo progress-so-far; sleep 30", check=False) + finally: + await ws.stop() + + assert "progress-so-far" in str(result.stdout) + assert "timed out" in str(result.stderr) + assert result.exit_status == 1 + + def _wall(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(Workspace, "_drops_privileges", lambda self: True) monkeypatch.setattr(Workspace, "_setpriv", lambda self: "/usr/bin/setpriv") diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index c4c8cae14..0317611ba 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -937,15 +937,26 @@ async def relay_stdin() -> None: finally: stdin.close() - async def drain_output(reader: asyncio.StreamReader, output: bytearray) -> None: - while chunk := await reader.read(65536): - output.extend(chunk) + async def relay_output( + reader: asyncio.StreamReader, writer: asyncssh.SSHWriter[bytes] + ) -> None: + """Forward the child's output as it is produced. + + Streamed, not accumulated: an agent watching a build wants the + lines while it runs, a session that never exits would otherwise + say nothing at all, and a command killed at the timeout still + keeps whatever it managed to print. + """ + try: + while chunk := await reader.read(65536): + writer.write(chunk) + await writer.drain() + except (asyncssh.Error, BrokenPipeError, ConnectionResetError): + pass - stdout_data = bytearray() - stderr_data = bytearray() stdin_task = asyncio.create_task(relay_stdin()) - stdout_task = asyncio.create_task(drain_output(stdout, stdout_data)) - stderr_task = asyncio.create_task(drain_output(stderr, stderr_data)) + stdout_task = asyncio.create_task(relay_output(stdout, process.stdout)) + stderr_task = asyncio.create_task(relay_output(stderr, process.stderr)) wait_task = asyncio.create_task(sub.wait()) channel_closed_task = asyncio.create_task(process.channel.wait_closed()) timed_out = False @@ -993,15 +1004,13 @@ async def drain_output(reader: asyncio.StreamReader, output: bytearray) -> None: if process.channel.is_closing(): return if timed_out: + # Whatever ran before the deadline has already been relayed; this + # only says why it stopped. process.stderr.write( f"workspace: command timed out after {_COMMAND_TIMEOUT:g}s\n".encode() ) process.exit(1) return - if stdout_data: - process.stdout.write(bytes(stdout_data)) - if stderr_data: - process.stderr.write(bytes(stderr_data)) process.exit(sub.returncode if sub.returncode is not None else 0) From 50b2acc9251c0cf6601d680cd2378c62f384d9e1 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:56:03 -0700 Subject: [PATCH 06/33] feat(environment): give a session the terminal it asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sessions always ran on pipes, so isatty() was false and every program that branches on it took its non-interactive path: no readline, no curses, no progress rendering, and a terminal task graded on behaviour a terminal would never produce. A pty request from the client was accepted by the protocol and then quietly ignored. Honour it: allocate a terminal pair, hand the child the slave as its three std fds, and relay the master to the channel. stderr merges into stdout, as it does on any tty. The size the client asked for is applied to the terminal rather than left at a default, and TERM goes into the session environment — sessions start from an exact environment, so without that deliberate step curses has no terminal description and fails outright. Verified through nsenter into the sandbox, not just locally: stty reports the requested geometry, stdin is a pts device, and a curses program starts and reads back the right size. --- hud/environment/tests/test_workspace.py | 24 +++++ hud/environment/workspace.py | 118 ++++++++++++++++++------ 2 files changed, 115 insertions(+), 27 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 47b517109..9fe9a428f 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -107,6 +107,30 @@ async def test_output_arrives_while_the_command_is_still_running(tmp_path: Path) assert elapsed < 2.0, f"first line took {elapsed:.1f}s — output is not streaming" +@pytest.mark.asyncio +async def test_a_session_that_asks_for_a_terminal_gets_one(tmp_path: Path) -> None: + """Programs branch on isatty: without a pty they take their batch path, so + a terminal task is graded on behaviour a terminal would never produce.""" + ws = Workspace(tmp_path / "root") + await ws.start() + try: + async with await _connect(ws) as conn: + with_pty = await conn.run( + "test -t 0 && test -t 1 && echo TTY || echo NOT_TTY; tput cols 2>/dev/null", + term_type="xterm-256color", + term_size=(120, 40), + check=False, + ) + without = await conn.run("test -t 1 && echo TTY || echo NOT_TTY", check=False) + finally: + await ws.stop() + + assert "TTY" in str(with_pty.stdout) and "NOT_TTY" not in str(with_pty.stdout) + # The size the client asked for reaches the terminal, not a default. + assert "120" in str(with_pty.stdout) + assert "NOT_TTY" in str(without.stdout) + + @pytest.mark.asyncio async def test_a_timed_out_command_keeps_what_it_printed( tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 0317611ba..c72d340b4 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -9,17 +9,23 @@ import os import shutil import socket +import struct import subprocess import sys import tempfile from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal import asyncssh from hud.utils.process import create_process_group_exec +if sys.platform != "win32": # the pty a session runs on has no Windows analogue + import fcntl + import pty + import termios + if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -168,6 +174,38 @@ def _env_argv(env: Mapping[str, str]) -> list[str]: return [env_bin, "-i", *(f"{k}={v}" for k, v in env.items())] +def _open_pty(process: asyncssh.SSHServerProcess[bytes]) -> tuple[int, int]: + """A terminal pair sized to what the client asked for: (master, slave).""" + master_fd, slave_fd = pty.openpty() + width, height, pixwidth, pixheight = process.get_terminal_size() + with contextlib.suppress(OSError): + fcntl.ioctl( + slave_fd, termios.TIOCSWINSZ, struct.pack("HHHH", height, width, pixwidth, pixheight) + ) + return master_fd, slave_fd + + +async def _pty_streams(master_fd: int) -> tuple[Any, asyncio.StreamReader]: + """Async ends of the terminal: something to write keystrokes to, and the + screen output to read. + + Both sides go through the event loop rather than blocking reads, so one + talkative program cannot stall the server, and ``drain`` gives the same + backpressure the pipe path has. + """ + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader() + await loop.connect_read_pipe( + lambda: asyncio.StreamReaderProtocol(reader), os.fdopen(master_fd, "rb", 0) + ) + # A dup so the read and write ends own their own file objects; closing one + # must not pull the terminal out from under the other. + transport, protocol = await loop.connect_write_pipe( + asyncio.streams.FlowControlMixin, os.fdopen(os.dup(master_fd), "wb", 0) + ) + return asyncio.StreamWriter(transport, protocol, None, loop), reader + + def _payload_argv(command: str | list[str] | None, env: Mapping[str, str]) -> list[str]: """The session itself: a login shell (or an exact argv) under ``env``.""" argv = _env_argv(env) @@ -818,10 +856,14 @@ def _session_env(self) -> dict[str, str] | None: async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> None: pid = await self.sandbox_pid() + # Sessions start from an exact environment, so a terminal's TERM has to + # be put there deliberately: without it curses and tput have no + # terminal description and fall back or fail outright. + session_env = {"TERM": process.term_type} if process.term_type else None argv = ( - self.shell_argv(process.command) + self.shell_argv(process.command, env=session_env) if pid is None - else self.enter_argv(pid, process.command) + else self.enter_argv(pid, process.command, env=session_env) ) if self._drops_privileges(): # The pre-drop processes (setpriv, bwrap) run as root; caller env @@ -832,6 +874,10 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No } else: proc_env = self._session_env() + if session_env: + # The sandboxed paths carry TERM in the argv's own `env -i`; a + # session running directly inherits this environment instead. + proc_env = {**(proc_env if proc_env is not None else os.environ), **session_env} if sys.platform == "win32": # On Windows, asyncio.create_subprocess_exec uses the ProactorEventLoop's @@ -906,36 +952,53 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No process.exit(result.returncode) return + # A client that asked for a pty gets one: the child's std fds are the + # terminal, so isatty() holds and curses/readline programs behave as + # they would in a terminal. stderr merges into stdout, as on any tty. + pty_pair = _open_pty(process) if process.term_type else None + child_fds: dict[str, Any] = ( + { + "stdin": asyncio.subprocess.PIPE, + "stdout": asyncio.subprocess.PIPE, + "stderr": asyncio.subprocess.PIPE, + } + if pty_pair is None + else {"stdin": pty_pair[1], "stdout": pty_pair[1], "stderr": pty_pair[1]} + ) try: sub = await create_process_group_exec( - *argv, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=str(self.root), - env=proc_env, + *argv, **child_fds, cwd=str(self.root), env=proc_env ) except FileNotFoundError as exc: + if pty_pair is not None: + os.close(pty_pair[0]) + os.close(pty_pair[1]) process.stderr.write(f"workspace: cannot spawn shell: {exc}\n".encode()) process.exit(127) return - stdin = sub.process.stdin - stdout = sub.stdout - stderr = sub.stderr - assert stdin is not None - assert stdout is not None - assert stderr is not None + if pty_pair is not None: + # The child holds the terminal now; this side keeps only the master. + os.close(pty_pair[1]) + stdin_writer, stdout_reader = await _pty_streams(pty_pair[0]) + stderr_reader = None + else: + stdin_writer = sub.process.stdin + stdout_reader = sub.stdout + stderr_reader = sub.stderr + assert stdin_writer is not None + assert stdout_reader is not None async def relay_stdin() -> None: try: while chunk := await process.stdin.read(65536): - stdin.write(chunk) - await stdin.drain() - except (asyncssh.Error, BrokenPipeError, ConnectionResetError): + stdin_writer.write(chunk) + await stdin_writer.drain() + except (asyncssh.Error, BrokenPipeError, ConnectionResetError, OSError): pass finally: - stdin.close() + with contextlib.suppress(Exception): + stdin_writer.close() async def relay_output( reader: asyncio.StreamReader, writer: asyncssh.SSHWriter[bytes] @@ -951,12 +1014,16 @@ async def relay_output( while chunk := await reader.read(65536): writer.write(chunk) await writer.drain() - except (asyncssh.Error, BrokenPipeError, ConnectionResetError): + except (asyncssh.Error, BrokenPipeError, ConnectionResetError, OSError): + # A pty master reads EIO once the child is gone: end of output, + # not a failure. pass stdin_task = asyncio.create_task(relay_stdin()) - stdout_task = asyncio.create_task(relay_output(stdout, process.stdout)) - stderr_task = asyncio.create_task(relay_output(stderr, process.stderr)) + # One stream on a terminal, where stderr shares the tty, two otherwise. + output_tasks = [asyncio.create_task(relay_output(stdout_reader, process.stdout))] + if stderr_reader is not None: + output_tasks.append(asyncio.create_task(relay_output(stderr_reader, process.stderr))) wait_task = asyncio.create_task(sub.wait()) channel_closed_task = asyncio.create_task(process.channel.wait_closed()) timed_out = False @@ -993,13 +1060,10 @@ async def relay_output( channel_closed_task, return_exceptions=True, ) - _, output_pending = await asyncio.wait( - (stdout_task, stderr_task), - timeout=1.0, - ) + _, output_pending = await asyncio.wait(output_tasks, timeout=1.0) for task in output_pending: task.cancel() - await asyncio.gather(stdout_task, stderr_task, return_exceptions=True) + await asyncio.gather(*output_tasks, return_exceptions=True) if process.channel.is_closing(): return From bfa5f7c441e1020447c832c1d23f532e91214753 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:06:52 -0700 Subject: [PATCH 07/33] =?UTF-8?q?fix(environment):=20complete=20the=20term?= =?UTF-8?q?inal=20=E2=80=94=20controlling=20tty=20and=20resize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps left by the pty itself. A session had terminal file descriptors but no controlling terminal, so /dev/tty could not be opened (ENXIO) and job control had no foreground process group: `less`, `vim` reading the terminal directly, and anything expecting Ctrl-C to reach a foreground job behaved unlike they would in a terminal. Claim it with `setsid -c` around the payload. Run there it is not a process-group leader, so it execs in place rather than forking, leaving the pid and exit status the caller waits on intact. And a window resize arrives from asyncssh as an exception raised by the stdin read, not as data — TerminalSizeChanged, which is not an asyncssh.Error and so was caught by nothing. The first resize escaped the relay coroutine and killed the session's keyboard for good. Handle it where it surfaces: resize the terminal and keep reading. Verified in the container: /dev/tty opens, tcgetpgrp reports the session's own process group as the foreground one, and curses still starts clean. --- hud/environment/tests/test_workspace.py | 23 +++++++ hud/environment/workspace.py | 79 ++++++++++++++++++++----- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 9fe9a428f..e7c72a2ff 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -131,6 +131,29 @@ async def test_a_session_that_asks_for_a_terminal_gets_one(tmp_path: Path) -> No assert "NOT_TTY" in str(without.stdout) +@pytest.mark.asyncio +async def test_a_resize_does_not_cost_the_session_its_keyboard(tmp_path: Path) -> None: + """asyncssh delivers a resize as an exception on the stdin read, and it is + not an asyncssh.Error — unhandled it escapes the relay and input stops.""" + ws = Workspace(tmp_path / "root") + await ws.start() + try: + async with await _connect(ws) as conn: + process = await conn.create_process( + "cat", term_type="xterm-256color", term_size=(80, 24) + ) + process.channel.change_terminal_size(132, 43) + await asyncio.sleep(0.2) + # Input still reaches the shell after the resize. + process.stdin.write("still-listening\n") + echoed = await asyncio.wait_for(process.stdout.readline(), 5) + process.channel.close() + finally: + await ws.stop() + + assert "still-listening" in echoed + + @pytest.mark.asyncio async def test_a_timed_out_command_keeps_what_it_printed( tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index c72d340b4..752108ab2 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -177,14 +177,34 @@ def _env_argv(env: Mapping[str, str]) -> list[str]: def _open_pty(process: asyncssh.SSHServerProcess[bytes]) -> tuple[int, int]: """A terminal pair sized to what the client asked for: (master, slave).""" master_fd, slave_fd = pty.openpty() - width, height, pixwidth, pixheight = process.get_terminal_size() - with contextlib.suppress(OSError): - fcntl.ioctl( - slave_fd, termios.TIOCSWINSZ, struct.pack("HHHH", height, width, pixwidth, pixheight) - ) + _set_winsize(slave_fd, *process.get_terminal_size()) return master_fd, slave_fd +def _set_winsize(fd: int, width: int, height: int, pixwidth: int, pixheight: int) -> None: + """Tell the terminal how big it is. + + Full-screen programs lay out against this and never re-measure, so a stale + size leaves them drawing to the wrong shape for the rest of the session. + """ + with contextlib.suppress(OSError): + fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", height, width, pixwidth, pixheight)) + + +def _ctty_argv() -> list[str]: + """Claim the session's terminal as its *controlling* terminal. + + tty file descriptors alone are not a terminal session: ``/dev/tty`` cannot + be opened, and job control has no foreground process group to signal. Run + inside the sandbox, ``setsid`` is not a process-group leader there and so + execs in place rather than forking, which keeps the pid and exit status + the caller is waiting on. Where the binary is absent (macOS ships none) + the session still gets a working tty, only without a ctty. + """ + setsid = shutil.which("setsid") + return [setsid, "-c"] if setsid else [] + + async def _pty_streams(master_fd: int) -> tuple[Any, asyncio.StreamReader]: """Async ends of the terminal: something to write keystrokes to, and the screen output to read. @@ -206,9 +226,11 @@ async def _pty_streams(master_fd: int) -> tuple[Any, asyncio.StreamReader]: return asyncio.StreamWriter(transport, protocol, None, loop), reader -def _payload_argv(command: str | list[str] | None, env: Mapping[str, str]) -> list[str]: +def _payload_argv( + command: str | list[str] | None, env: Mapping[str, str], *, ctty: bool = False +) -> list[str]: """The session itself: a login shell (or an exact argv) under ``env``.""" - argv = _env_argv(env) + argv = [*(_ctty_argv() if ctty else []), *_env_argv(env)] if isinstance(command, str): return [*argv, "bash", "-lc", command] if command is None: @@ -555,6 +577,7 @@ def bwrap_argv( env: Mapping[str, str] | None = None, inherit_host_env: bool = True, info_fd: int | None = None, + tty: bool = False, ) -> list[str]: """Argv that runs ``command`` inside bwrap. Raises if bwrap unavailable. @@ -590,7 +613,7 @@ def bwrap_argv( argv.extend(m.to_bwrap_args()) argv.extend(["--chdir", target_cwd]) argv.append("--") - argv.extend(_payload_argv(command, full_env)) + argv.extend(_payload_argv(command, full_env, ctty=tty)) return argv def enter_argv( @@ -600,6 +623,7 @@ def enter_argv( *, cwd: str | None = None, env: Mapping[str, str] | None = None, + tty: bool = False, ) -> list[str]: """Argv that runs ``command`` inside the sandbox *pid* belongs to. @@ -631,7 +655,7 @@ def enter_argv( # Unlike the bwrap path, the drop goes *inside*: joining namespaces # needs the privileges the dropped uid does not have. argv.extend(self._drop_argv()) - argv.extend(_payload_argv(command, self._full_env(env))) + argv.extend(_payload_argv(command, self._full_env(env), ctty=tty)) return argv def _full_env(self, env: Mapping[str, str] | None = None) -> dict[str, str]: @@ -752,6 +776,7 @@ def shell_argv( *, cwd: str | None = None, env: Mapping[str, str] | None = None, + tty: bool = False, ) -> list[str]: """Per-session shell argv (bwrap'd if available, else host shell). @@ -769,9 +794,11 @@ def shell_argv( # the same minimal environment as the non-bwrap dropped shell, # keeping explicit per-call overrides. walled_env = {**(self._session_env() or {}), **(env or {})} - argv = self.bwrap_argv(inner, cwd=cwd, env=walled_env, inherit_host_env=False) + argv = self.bwrap_argv( + inner, cwd=cwd, env=walled_env, inherit_host_env=False, tty=tty + ) else: - argv = self.bwrap_argv(inner, cwd=cwd, env=env) + argv = self.bwrap_argv(inner, cwd=cwd, env=env, tty=tty) elif command is not None: argv = ["bash", "-lc", command] else: @@ -859,11 +886,13 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No # Sessions start from an exact environment, so a terminal's TERM has to # be put there deliberately: without it curses and tput have no # terminal description and fall back or fail outright. - session_env = {"TERM": process.term_type} if process.term_type else None + term_type = process.term_type + wants_tty = bool(term_type) + session_env = {"TERM": term_type} if term_type else None argv = ( - self.shell_argv(process.command, env=session_env) + self.shell_argv(process.command, env=session_env, tty=wants_tty) if pid is None - else self.enter_argv(pid, process.command, env=session_env) + else self.enter_argv(pid, process.command, env=session_env, tty=wants_tty) ) if self._drops_privileges(): # The pre-drop processes (setpriv, bwrap) run as root; caller env @@ -955,7 +984,7 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No # A client that asked for a pty gets one: the child's std fds are the # terminal, so isatty() holds and curses/readline programs behave as # they would in a terminal. stderr merges into stdout, as on any tty. - pty_pair = _open_pty(process) if process.term_type else None + pty_pair = _open_pty(process) if wants_tty else None child_fds: dict[str, Any] = ( { "stdin": asyncio.subprocess.PIPE, @@ -991,7 +1020,25 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No async def relay_stdin() -> None: try: - while chunk := await process.stdin.read(65536): + while True: + try: + chunk = await process.stdin.read(65536) + except asyncssh.TerminalSizeChanged as resized: + # A resize arrives as an exception on the read rather + # than as data. It is not an asyncssh.Error, so left + # alone it would escape this coroutine and take the + # session's keyboard with it. + if pty_pair is not None: + _set_winsize( + pty_pair[0], + resized.width, + resized.height, + resized.pixwidth, + resized.pixheight, + ) + continue + if not chunk: + break stdin_writer.write(chunk) await stdin_writer.drain() except (asyncssh.Error, BrokenPipeError, ConnectionResetError, OSError): From c2e51109be45b300c684735d214c2f106892368c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:10:36 -0700 Subject: [PATCH 08/33] refactor(environment): make shell_argv honour env and tty unsandboxed too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unsandboxed branch built a bare `bash -lc`, so `env` and `tty` were accepted and then dropped: a caller asking for a terminal or passing variables got neither, silently, depending only on whether bwrap happened to be usable. It surfaced as a terminal session with no TERM, which leaves curses with no terminal description at all. Run the same payload the sandboxed forms do. `cwd` stays the one argument the unsandboxed form cannot honour — there is no mount namespace to chdir into — and the docstring now says so rather than leaving the caller to find out. This also removes the TERM patch-up the handler was doing on the side, so one mechanism carries the session environment on every path. --- hud/environment/workspace.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 752108ab2..76a82661a 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -782,6 +782,11 @@ def shell_argv( With ``shell_uid`` set and the serving process running as root, the whole session is wrapped in ``setpriv`` to drop to that uid. + + ``cwd`` is the one argument the unsandboxed form cannot honour: there + is no mount namespace to ``--chdir`` into, so the session runs wherever + the caller starts the process — which for a :class:`Workspace` is its + root, the same path ``_guest_path`` takes when bwrap is unavailable. """ if sys.platform == "win32": if command is not None: @@ -799,16 +804,14 @@ def shell_argv( ) else: argv = self.bwrap_argv(inner, cwd=cwd, env=env, tty=tty) - elif command is not None: - argv = ["bash", "-lc", command] else: - argv = ["bash", "-l"] + # The same payload the sandboxed forms run. Built here too rather + # than left as a bare shell, so that ``env`` and ``tty`` mean the + # same thing however the session is placed — and so the session + # env reaches the shell only *after* any drop: an LD_PRELOAD in it + # must never be in the environment of the root-run setpriv. + argv = _payload_argv(command, self._full_env(env), ctty=tty) if self._drops_privileges(): - if self._bwrap is None: - # The session env (self.env + per-call overrides) is injected - # only *after* the drop: vars like LD_PRELOAD in it must never - # be in the environment of the root-run setpriv itself. - argv = [*_env_argv(self._full_env(env)), *argv] argv = [*self._drop_argv(), *argv] return argv @@ -903,10 +906,6 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No } else: proc_env = self._session_env() - if session_env: - # The sandboxed paths carry TERM in the argv's own `env -i`; a - # session running directly inherits this environment instead. - proc_env = {**(proc_env if proc_env is not None else os.environ), **session_env} if sys.platform == "win32": # On Windows, asyncio.create_subprocess_exec uses the ProactorEventLoop's From e79a3534cf75fb4acdec791eb6dc034aa7617153 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:18:35 -0700 Subject: [PATCH 09/33] fix(environment): keep the harness out of the session's environment and /tmp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things told an agent what was running it, and one of them handed over a key while doing so. The serving process's environment carried through to sessions whole, so anything HUD was configured with arrived in the agent's `env` — the version check flag this branch added, and on a deployed placement an API key. Filter HUD's own variables out of what a session inherits. What the task or the caller declares is layered on afterwards and still arrives, HUD-shaped name or not: this drops only what the serving process happened to hold. Session key material went to a mkdtemp under /tmp, which is the *task's* /tmp: the agent could read the private key to its own shell, the host key and authorized_keys, in a directory named after the harness. Let a caller place credentials somewhere it masks, and have Harbor put them under /hud. Outside the served root was never the same as out of the session's reach. Verified from inside the container: the agent's environment has no HUD_ anything, /tmp is empty, and a search for key material finds none. --- hud/environment/tests/test_workspace.py | 25 ++++++++++++++++-- hud/environment/workspace.py | 34 +++++++++++++++++++++---- integrations/harbor/_adapt.py | 9 ++++++- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index e7c72a2ff..ae9ccbec7 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -223,11 +223,32 @@ def test_bwrap_drops_host_env_when_walled(tmp_path: Path, monkeypatch: pytest.Mo def test_bwrap_inherits_host_env_when_not_walled( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setenv("HUD_SENTINEL", "visible") + monkeypatch.setenv("SENTINEL", "visible") ws = Workspace(tmp_path / "root") monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") argv = ws.bwrap_argv(["bash", "-lc", "true"]) - assert _sandbox_env(argv)["HUD_SENTINEL"] == "visible" + assert _sandbox_env(argv)["SENTINEL"] == "visible" + + +def test_the_harness_own_configuration_never_reaches_a_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """HUD's variables in the serving process are a credential leak where they + hold a key, and a tell everywhere else — an agent that finds HUD_ anything + knows what is running it.""" + monkeypatch.setenv("HUD_API_KEY", "super-secret") + monkeypatch.setenv("HUD_SKIP_VERSION_CHECK", "1") + monkeypatch.setenv("ORDINARY", "kept") + # What the task itself declares is the task's, HUD-shaped name or not. + ws = Workspace(tmp_path / "root", env={"HUD_TASK_DECLARED": "mine"}) + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + + for argv in (ws.shell_argv("echo hi"), ws.bwrap_argv(["true"]), ws.enter_argv(7, "echo hi")): + session_env = _sandbox_env(argv) + assert "HUD_API_KEY" not in session_env + assert "HUD_SKIP_VERSION_CHECK" not in session_env + assert session_env["ORDINARY"] == "kept" + assert session_env["HUD_TASK_DECLARED"] == "mine" #: Options bubblewrap gained after 0.4, the newest release on distros still in diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 76a82661a..f8d1d3eeb 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -165,6 +165,19 @@ def to_bwrap_args(self) -> list[str]: _SANDBOX_READY = b"ready\n" +def _without_harness_config(environ: Mapping[str, str]) -> dict[str, str]: + """The serving process's environment, minus HUD's own configuration. + + A session runs in the *task's* environment, not the harness's. HUD's + variables reaching it are a credential leak where they hold an API key, + and a tell everywhere else: an agent that finds ``HUD_`` anything in its + environment knows exactly what is running it. Variables the task or the + caller declare are layered on afterwards and are unaffected — this drops + only what the serving process happened to be configured with. + """ + return {key: value for key, value in environ.items() if not key.startswith("HUD_")} + + def _env_argv(env: Mapping[str, str]) -> list[str]: """``env -i`` and its assignments: an exact environment for what follows. @@ -275,10 +288,12 @@ def __init__( track_files: bool = False, shell_uid: int | None = None, require_isolation: bool = False, + credentials_dir: Path | str | None = None, ) -> None: self.root: Path = Path(root).resolve() # Per-instance credential dir, materialized lazily (see _credentials_dir). self._cred_dir: Path | None = None + self._configured_cred_dir = Path(credentials_dir) if credentials_dir else None # Path the root is mounted at inside the sandbox (and the default cwd). # Defaults to /workspace; set to the root's real path for callers that @@ -591,7 +606,7 @@ def bwrap_argv( if self._bwrap is None: raise RuntimeError("bwrap not available on this host") target_cwd = cwd if cwd is not None else self._guest_path - base_env = dict(os.environ) if inherit_host_env else {} + base_env = _without_harness_config(os.environ) if inherit_host_env else {} full_env = {**base_env, **self.env, **(env or {})} argv: list[str] = [ self._bwrap, @@ -662,11 +677,11 @@ def _full_env(self, env: Mapping[str, str] | None = None) -> dict[str, str]: """The environment a session starts from. Dropped sessions get the minimal one built for the wall; otherwise the - serving process's environment carries through, as it always has. + serving process's environment carries through, less HUD's own. """ if self._drops_privileges(): return {**(self._session_env() or {}), **(env or {})} - return {**os.environ, **self.env, **(env or {})} + return {**_without_harness_config(os.environ), **self.env, **(env or {})} def _drop_argv(self) -> list[str]: """The ``setpriv`` prefix that drops to ``shell_uid``, if it applies.""" @@ -823,10 +838,19 @@ def _credentials_dir(self) -> Path: ``mkdtemp`` creates a fresh 0700 directory with an unpredictable name atomically, so a local user can't pre-place a symlink at the path to - redirect the private keys the server writes here. + redirect the private keys the server writes here. A caller that masks + part of the filesystem from sessions should pass ``credentials_dir`` + pointing inside it: outside the served root is not the same as out of + the session's reach, and these are the keys to the session itself. """ if self._cred_dir is None: - self._cred_dir = Path(tempfile.mkdtemp(prefix="hud-workspace-creds-")) + if self._configured_cred_dir is not None: + self._configured_cred_dir.mkdir(parents=True, exist_ok=True) + self._configured_cred_dir.chmod(0o700) + self._cred_dir = self._configured_cred_dir + else: + # Named for what it holds, not for what put it there. + self._cred_dir = Path(tempfile.mkdtemp(prefix="ssh-")) return self._cred_dir def _load_or_generate_host_key(self) -> tuple[asyncssh.SSHKey, str]: diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 2721c30f0..a7e9c5dc3 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -73,6 +73,12 @@ VERIFIER_LOGS = LOGS / "verifier" TESTS = Path("/tests") +#: Where the adaptation layer keeps itself, and the one path agent sessions +#: never see. The session's own key material belongs here rather than in a +#: temp dir: /tmp is the task's, so keys left there are both readable by the +#: graded party and a signpost saying what is running it. +HUD_ROOT = Path("/hud") + #: Task config is untrusted: these bound what may reach a generated directive. _DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*") @@ -440,10 +446,11 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E # dir are throwaway tmpfs here, while grading runs outside this # namespace and sees the real ones. mounts=( - Mount("tmpfs", dst="/hud"), + Mount("tmpfs", dst=str(HUD_ROOT)), Mount("tmpfs", dst=str(TESTS)), Mount("tmpfs", dst=str(VERIFIER_LOGS)), ), + credentials_dir=HUD_ROOT / "session-keys", track_files=False if rooted_at_filesystem else None, # The agent phase's own variables, scoped to its sessions. env=policy["agent_env"], From c5da75d148f62b1bcea9f558b53a0aa6f54fd7bf Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:37:55 -0700 Subject: [PATCH 10/33] fix(integrations): let the agent phase not contain the grading paths at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /tests and the verdict directory were masked with a tmpfs, so the agent saw them as empty directories. The base image ships both empty and an earlier rollout left them behind, so they were there for the whole agent phase — two directories the task's own image would not have, announcing that something is holding the assertions and the verdict back. Harbor does not have them during its agent phase either; it uploads the verifier when it runs it. Do the same: remove both when the task starts and again when it ends, and let _grade lay them down in between for as long as the verifier needs. Removing rather than emptying is the point — an empty directory is the signal. Where the serve process cannot remove them (not root, and they sit at the filesystem root) it empties them instead, which is what the mask achieved. /hud keeps its mask: it has to exist for the whole rollout, being what serves. This drops the stricter-than-Harbor property the masks gave, that a process the agent leaves running cannot reach the assertions or the verdict while the verifier works. That exposure is Harbor's own in shared mode, and its answer is environment_mode = "separate", which this integration still refuses rather than grading as if it were shared. --- integrations/harbor/_adapt.py | 50 ++++++++++++++-------- integrations/harbor/tests/test_contract.py | 29 +++++++++++-- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index a7e9c5dc3..42e41248e 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -439,17 +439,12 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E ), # Masks go in ``mounts``, which bwrap applies *after* the workspace # bind — as system mounts they would be re-covered when the guest - # path is ``/`` (an image with no WORKDIR). The graded party's - # namespace must not contain the grading material or the verdict: - # the baked tests and serving venv (/hud), the verifier Harbor only - # uploads once the agent is done (/tests), and the verifier's output - # dir are throwaway tmpfs here, while grading runs outside this - # namespace and sees the real ones. - mounts=( - Mount("tmpfs", dst=str(HUD_ROOT)), - Mount("tmpfs", dst=str(TESTS)), - Mount("tmpfs", dst=str(VERIFIER_LOGS)), - ), + # path is ``/`` (an image with no WORKDIR). Only the harness's own + # tree is masked: it has to exist for the whole rollout, since it is + # what is serving. ``/tests`` and the verdict are handled by not + # existing during the agent phase at all (:func:`_hide_grading_dirs`), + # which is what Harbor does — it uploads the verifier when it runs it. + mounts=(Mount("tmpfs", dst=str(HUD_ROOT)),), credentials_dir=HUD_ROOT / "session-keys", track_files=False if rooted_at_filesystem else None, # The agent phase's own variables, scoped to its sessions. @@ -474,16 +469,18 @@ def _register(env: Environment, task_dir: Path, workdir: Path, workspace: Worksp description=config.task.description or f"Harbor task {task_dir.name}", ) async def _run_harbor_task() -> AsyncGenerator[Any, Any]: - # Harbor uploads /tests when it runs the verifier, so the agent phase - # never contains the assertions it is graded on; :func:`_grade` lays - # the baked copy down at that same point. One adapted image serves a - # whole group and may serve many rollouts, so the directory is emptied - # afterwards: no agent ever sees another task's tests. + # Harbor uploads /tests when it runs the verifier, so its agent phase + # never contains the assertions it is graded on — nor the verdict dir. + # Adapted images ship both (empty), and an earlier rollout in this + # container leaves them behind, so the agent phase starts by removing + # them and ends the same way: :func:`_grade` lays them down in between, + # for as long as the verifier needs them. + _hide_grading_dirs() try: answer = yield (task_dir / "instruction.md").read_text(encoding="utf-8") yield await _grade(task_dir, workdir, answer) finally: - _reset_dir(TESTS) + _hide_grading_dirs() # Harbor's agent phase is one continuous session, so a service the # agent starts is still running when the verifier looks for it — # and is gone before the next rollout, which this container may @@ -498,6 +495,25 @@ def _sync_tests(task_dir: Path) -> None: _copy_task_content(child, TESTS / child.name) +def _hide_grading_dirs() -> None: + """Leave no ``/tests`` or verdict directory for the agent phase to find. + + Removed rather than emptied: an empty directory the base image does not + have is itself a signal, and these two are the harness's whole visible + footprint outside its own tree. Where the serve process cannot remove them + — not root, and they sit at the filesystem root — emptying is the fallback, + which is what the mask used to achieve. + """ + for path in (TESTS, VERIFIER_LOGS): + try: + shutil.rmtree(path) + except FileNotFoundError: + continue + except OSError: + LOGGER.debug("could not remove %s; emptying it instead", path) + _reset_dir(path) + + def _reset_dir(path: Path) -> None: """Leave *path* an existing, empty directory, whatever it was before. diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index 6d2810882..0686541c5 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -449,6 +449,26 @@ def test_an_invalid_task_toml_is_an_error_not_a_default(tmp_path) -> None: harbor.load(tmp_path) +def test_grading_directories_do_not_exist_during_the_agent_phase(tmp_path, monkeypatch) -> None: + """The image ships /tests and the verdict dir empty, and a previous rollout + leaves them behind — either way the agent phase must not find them.""" + from integrations.harbor import _adapt + + tests, verdict = tmp_path / "tests", tmp_path / "logs" / "verifier" + for stale in (tests, verdict): + stale.mkdir(parents=True) + (tests / "test.sh").write_text("the assertions", encoding="utf-8") + monkeypatch.setattr(_adapt, "TESTS", tests) + monkeypatch.setattr(_adapt, "VERIFIER_LOGS", verdict) + + _adapt._hide_grading_dirs() + + assert not tests.exists() + assert not verdict.exists() + # /logs itself is Harbor's and stays: the agent's own answer is written there. + assert verdict.parent.exists() + + async def test_masks_are_applied_after_the_workspace_bind(tmp_path, monkeypatch) -> None: # bwrap applies ``mounts`` after the workspace bind; as system mounts the # masks would be re-covered when the guest path is "/" (no WORKDIR). @@ -473,10 +493,11 @@ def record(*args, **kwargs): (workspace,) = built masked = [m.dst for m in workspace.mounts] assert "/hud" in masked - assert "/logs/verifier" in masked - # Harbor uploads the verifier only once the agent is done, so the graded - # party never has the assertions it is graded on. - assert "/tests" in masked + # The assertions and the verdict are kept from the agent phase by not + # existing during it, not by masking: an empty directory the base image + # does not have is itself a signal. + assert "/tests" not in masked + assert "/logs/verifier" not in masked assert [m.dst for m in workspace._system_mounts] == ["/", "/proc", "/dev"] From 51c84a1eef14c5a636c5a2036536ab17d9d2b518 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:27:39 -0700 Subject: [PATCH 11/33] fix(environment): map the container's ids into the sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bwrap maps a single id when left to create the user namespace alone, so every file owned by anyone else is nobody inside it: unreadable, unwritable, not even chownable by the sandbox's own root. For an image whose task runs as a non-root user that is most of the filesystem — the agent cannot write its own working directory — and no session can drop to an id the map does not contain, which made setpriv useless inside a sandbox. Install the map from out here instead, over bwrap's --userns-block-fd: the sandbox is held at its own creation until its ids are known. Writing a full range needs CAP_SETUID and CAP_SETGID in this namespace, which a container's root has by default — no added capability, no newuidmap, and both flags are old enough for the bubblewrap on bullseye. Where the kernel refuses, the fallback is the single id bwrap would have mapped: a narrow map is workable, an absent one is not. Also lets a caller keep the workspace root's ownership as the image staged it, rather than handing it to the session's uid. --- hud/environment/workspace.py | 68 ++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index f8d1d3eeb..bc0ed9772 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -164,6 +164,9 @@ def to_bwrap_args(self) -> list[str]: _SANDBOX_HOLDER = ["sh", "-c", "echo ready; exec sleep 2147483647"] _SANDBOX_READY = b"ready\n" +#: What the sandbox's user namespace maps: the container's ids, unchanged. +_FULL_ID_RANGE = "0 0 65536" + def _without_harness_config(environ: Mapping[str, str]) -> dict[str, str]: """The serving process's environment, minus HUD's own configuration. @@ -187,6 +190,36 @@ def _env_argv(env: Mapping[str, str]) -> list[str]: return [env_bin, "-i", *(f"{k}={v}" for k, v in env.items())] +def _map_identities(pid: int) -> None: + """Give the sandbox the whole id space, not just the id that created it. + + Left to itself bwrap maps one id, so every file owned by anyone else is + ``nobody`` inside — unreadable, unwritable, not even chownable by the + sandbox's own root. That is most of an image whose task runs as a non-root + user: the agent cannot write its own working directory, and no session can + drop to an id the map does not contain. Writing the map from out here + needs CAP_SETUID/CAP_SETGID in *this* namespace, which a container's root + holds by default. Where the kernel refuses, fall back to the single id + bwrap would have mapped: a narrow map is workable, an absent one is not. + """ + proc = Path(f"/proc/{pid}") + with contextlib.suppress(OSError): + (proc / "setgroups").write_text("allow") + for name, own in (("uid_map", os.geteuid()), ("gid_map", os.getegid())): + try: + (proc / name).write_text(_FULL_ID_RANGE) + except OSError: + if name == "gid_map": + # Giving up supplementary groups is the kernel's price for + # writing gid_map without the capability. + with contextlib.suppress(OSError): + (proc / "setgroups").write_text("deny") + try: + (proc / name).write_text(f"{own} {own} 1") + except OSError: + LOGGER.warning("could not map ids into the sandbox (%s)", name) + + def _open_pty(process: asyncssh.SSHServerProcess[bytes]) -> tuple[int, int]: """A terminal pair sized to what the client asked for: (master, slave).""" master_fd, slave_fd = pty.openpty() @@ -289,6 +322,7 @@ def __init__( shell_uid: int | None = None, require_isolation: bool = False, credentials_dir: Path | str | None = None, + hand_over_root: bool = True, ) -> None: self.root: Path = Path(root).resolve() # Per-instance credential dir, materialized lazily (see _credentials_dir). @@ -326,6 +360,9 @@ def __init__( self._ssh_port = port self._ssh_user = user self._shell_uid = shell_uid + # Whether the root is chowned to shell_uid at start. Off where the + # image staged it already: whose it is, is the image's statement. + self._hand_over_root = hand_over_root if require_isolation and self._bwrap is None: raise RuntimeError( "isolation was required but bwrap cannot sandbox here: install " @@ -418,7 +455,8 @@ def _prepare_runtime(self) -> None: # author's job, done where it's cheap and scoped: at build time # (`COPY --chown`) or in task setup over just what was staged. assert self._shell_uid is not None - os.lchown(self.root, self._shell_uid, self._shell_uid) + if self._hand_over_root: + os.lchown(self.root, self._shell_uid, self._shell_uid) self._host_key, self._host_pubkey_str = self._load_or_generate_host_key() self._authorized_keys_path = self._ensure_authorized_keys_file() self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -592,6 +630,7 @@ def bwrap_argv( env: Mapping[str, str] | None = None, inherit_host_env: bool = True, info_fd: int | None = None, + userns_block_fd: int | None = None, tty: bool = False, ) -> list[str]: """Argv that runs ``command`` inside bwrap. Raises if bwrap unavailable. @@ -611,7 +650,9 @@ def bwrap_argv( argv: list[str] = [ self._bwrap, "--die-with-parent", - "--unshare-user-try", + # Blocking means this side installs the map, so the namespace has + # to be ours to map: --unshare-user, not the best-effort form. + "--unshare-user" if userns_block_fd is not None else "--unshare-user-try", "--unshare-pid", "--unshare-ipc", "--unshare-uts", @@ -621,6 +662,8 @@ def bwrap_argv( argv.append("--unshare-net") if info_fd is not None: argv.extend(["--info-fd", str(info_fd)]) + if userns_block_fd is not None: + argv.extend(["--userns-block-fd", str(userns_block_fd)]) for m in self._system_mounts: argv.extend(m.to_bwrap_args()) argv.extend(["--bind", str(self.root), self._guest_path]) @@ -727,24 +770,35 @@ async def _start_sandbox(self) -> int: and bwrap is the only party that knows which of its forks that is. """ read_fd, write_fd = os.pipe() + block_read, block_write = os.pipe() try: os.set_inheritable(write_fd, True) - argv = self.bwrap_argv(_SANDBOX_HOLDER, info_fd=write_fd) + os.set_inheritable(block_read, True) + argv = self.bwrap_argv(_SANDBOX_HOLDER, info_fd=write_fd, userns_block_fd=block_read) self._sandbox = await asyncio.create_subprocess_exec( *argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - pass_fds=(write_fd,), + pass_fds=(write_fd, block_read), ) os.close(write_fd) - write_fd = -1 + os.close(block_read) + write_fd = block_read = -1 loop = asyncio.get_running_loop() raw = await asyncio.wait_for(loop.run_in_executor(None, os.read, read_fd, 4096), 30.0) + if raw: + # The sandbox is held at its own creation until this is + # written: nothing runs in it, and nothing can join it, before + # it knows who its ids are. + _map_identities(int(json.loads(raw)["child-pid"])) + os.write(block_write, b"\n") finally: os.close(read_fd) - if write_fd != -1: - os.close(write_fd) + os.close(block_write) + for stray in (write_fd, block_read): + if stray != -1: + os.close(stray) if not raw: raise RuntimeError(f"the sandbox holder did not start: {await self._sandbox_error()}") pid = int(json.loads(raw)["child-pid"]) From 5c094e738d80671961707839d94877c95488bfad Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:27:54 -0700 Subject: [PATCH 12/33] fix(integrations): run each phase as the identity declared for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The layer restored the image's USER for the whole container, which is every process in it — including the one serving. That identity is a statement about the task's *phases*: Harbor runs the agent and the verifier as it, and its own harness is not subject to it. Ours serves from inside the image, so the directive demoted the harness too, and a non-root harness can create neither /tests at the filesystem root nor its own state under /hud. Hence the pre-created, chowned-at-build directories, and hence a rollout that could not start at all once the session keys moved under /hud. Apply it where each phase runs instead: agent sessions drop with shell_uid, the verifier with setpriv at its own invocation, and the serving process stays root. Per phase, because Harbor's are — a task may hand the agent a restricted account and still verify as root, which one USER directive cannot express. Each falls back to the image's own USER, recorded into the layer because the build context is gone by serve time. Both carry a home: dropping ids alone leaves a phase pointed at the harness's, which it cannot write, and Harbor's verifiers commonly install their own tooling. The workspace root keeps the ownership the image gave it. This only works because the sandbox now maps the container's whole id space; with bwrap's single id, setpriv could not reach the declared user at all. --- integrations/harbor/_adapt.py | 123 ++++++++++++++++++--- integrations/harbor/_load.py | 14 ++- integrations/harbor/tests/test_contract.py | 41 +++++-- 3 files changed, 153 insertions(+), 25 deletions(-) diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 42e41248e..486638be0 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -32,10 +32,12 @@ from __future__ import annotations import asyncio +import contextlib import json import logging import math import os +import pwd import re import shlex import shutil @@ -199,6 +201,69 @@ """ +def _validated_user(declared: str | int | None, source_user: str | None) -> str | None: + """A phase's identity: what it declared, else the image's own ``USER``.""" + user = str(declared) if declared is not None else source_user + if user is None: + return None + # Untrusted input: the task config and the image's own Dockerfile. + if not all(_DOCKER_USER.fullmatch(part) for part in user.split(":", 1)): + raise ValueError(f"declared user {user!r} is not a usable user[:group]") + return user + + +def phase_uid(task_dir: Path, role: str) -> int | None: + """The id *role* runs as in this image, or None to run as the harness does. + + Harbor runs the agent and the verifier as the identity the task declares + for each, and its own harness is not subject to either. Serving from + inside the image, a ``USER`` directive would demote the harness too — + leaving it unable to create ``/tests`` at the filesystem root, or its own + state under ``/hud``. So the identity is applied where the phase runs. + + A name the image does not have is an error rather than a fall back to + root: running a phase with more privilege than the task granted is the + thing this exists to prevent. + """ + user = _validated_user(workspace_policy(task_dir)[f"{role}_user"], _image_user(task_dir)) + if user is None: + return None + name = user.split(":", 1)[0] + if name.isdigit(): + return int(name) + try: + return pwd.getpwnam(name).pw_uid + except KeyError as error: + raise ValueError(f"declared user {user!r} does not exist in this image") from error + + +def phase_home(uid: int | None) -> str | None: + """*uid*'s home, for a phase that is about to drop to it. + + Dropping ids without one leaves the phase pointed at the harness's home, + which it cannot write — and a verifier that installs its own tooling + (Harbor's commonly do) fails on that alone. + """ + if uid is None: + return None + with contextlib.suppress(KeyError): + return pwd.getpwuid(uid).pw_dir + return None + + +def _image_user(task_dir: Path) -> str | None: + """The ``USER`` the adapted image recorded for itself, if any. + + Read from the layer rather than the Dockerfile: by serve time the build + context is gone, and the image's own declaration still has to be honoured + for a task whose ``task.toml`` names no user of its own. + """ + recorded = HUD_ROOT / "image-user" + with contextlib.suppress(OSError): + return recorded.read_text(encoding="utf-8").strip() or None + return None + + def _declared_directives(task_dir: Path, source_user: str | None) -> str: """The task's declared environment, working dir and user as Dockerfile directives. @@ -228,18 +293,10 @@ def _declared_directives(task_dir: Path, source_user: str | None) -> str: lines.append(f"ENV {key}={literal}") if policy["workdir"]: lines.append(f"WORKDIR {json.dumps(policy['workdir'])}") - user = policy["user"] if policy["user"] is not None else source_user - lines.append("RUN mkdir -p /tests /logs/verifier") - if user is not None: - user = str(user) - # Docker's operand is ``user[:group]``; both parts are validated — - # the task config and Dockerfile are untrusted input. - if not all(_DOCKER_USER.fullmatch(part) for part in user.split(":", 1)): - raise ValueError(f"declared user {user!r} is not a usable user[:group]") - # Grading writes /tests and /logs, which the runtime user must own — - # handed over here, as root, before the identity switches back. - lines.append(f"RUN chown -R {user} /tests /logs") - lines.append(f"USER {user}") + # Validated at build, where a bad task config should fail, but applied per + # phase at runtime rather than as a USER directive — see :func:`phase_uid`. + for role in ("agent", "verifier"): + _validated_user(policy[f"{role}_user"], source_user) return "".join(f"{line}\n" for line in lines) @@ -344,6 +401,11 @@ def _write_context( "__PYTHON__", SERVING_PYTHON ), ) + # The image's own USER, kept for serve time: the build context is gone by + # then, and a task that declares no user of its own still runs its phases + # as whatever the image said. + if (image_user := final_stage(dockerfile).user) is not None: + _write(hud_dir / "image-user", f"{image_user}\n") layer = ( _LAYER.replace("__ENV_NAME__", env_name) .replace("__PYTHON__", SERVING_PYTHON) @@ -446,9 +508,17 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E # which is what Harbor does — it uploads the verifier when it runs it. mounts=(Mount("tmpfs", dst=str(HUD_ROOT)),), credentials_dir=HUD_ROOT / "session-keys", + # The agent's declared identity, not the harness's: the serving + # process stays root so it can place and remove the grading + # directories and keep its own state under /hud. + shell_uid=(agent_uid := phase_uid(task_dirs[0], "agent")), + # Whose the workspace is, is the image's statement — Harbor does not + # re-own it for the agent, so neither does this. + hand_over_root=False, track_files=False if rooted_at_filesystem else None, - # The agent phase's own variables, scoped to its sessions. - env=policy["agent_env"], + # The agent phase's own variables, scoped to its sessions. A dropped + # session would otherwise be pointed at the harness's home. + env={**({"HOME": home} if (home := phase_home(agent_uid)) else {}), **policy["agent_env"]}, network=policy["network"], # Always: the sandbox is what keeps the baked tests and the serving # venv out of the graded party's reach — an unsandboxed fallback @@ -546,6 +616,29 @@ async def _grade(task_dir: Path, workdir: Path, answer: Any) -> dict[str, Any]: argv = [str(test_sh)] config = TaskConfig.read(task_dir) + verifier_env: dict[str, str] = {} + verifier_uid = phase_uid(task_dir, "verifier") + setpriv = shutil.which("setpriv") + if verifier_uid is not None and setpriv is not None and os.geteuid() == 0: + # Root laid the grading directories down, so hand them over before + # dropping: a verifier that cannot write /logs/verifier cannot produce + # the verdict it is being run to produce. The verdict directory is made + # here rather than by the grading helper, since it must exist first. + _reset_dir(VERIFIER_LOGS) + for root_path in (TESTS, VERIFIER_LOGS): + for target in (root_path, *root_path.rglob("*")): + os.lchown(target, verifier_uid, verifier_uid) + if home := phase_home(verifier_uid): + verifier_env["HOME"] = home + argv = [ + setpriv, + "--reuid", + str(verifier_uid), + "--regid", + str(verifier_uid), + "--clear-groups", + *argv, + ] if not config.network("verifier"): # The verifier runs outside the agent sandbox, so its declared # isolation needs its own network namespace. @@ -577,7 +670,7 @@ async def run_tests() -> ProcessGroup: return await create_process_group_exec( *argv, cwd=workdir, - env={**os.environ, **config.verifier.env}, + env={**os.environ, **verifier_env, **config.verifier.env}, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py index c14bc429f..8433465f5 100644 --- a/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -118,9 +118,19 @@ def network(self, role: str) -> bool: """ return "no-network" not in (self.environment.network_mode, self.phase(role).network_mode) + def phase_user(self, role: str) -> str | int | None: + """The identity *role* runs as, if the task names one for it. + + Per phase, because Harbor's are: a task may hand the agent a + restricted account and still verify as root. They collapsed into one + value while the identity was a ``USER`` directive, which an image has + only one of; applied per phase, each keeps what it declared. + """ + return self.phase(role).user + @property def user(self) -> str | int | None: - """The identity the task's phases run as, if it names one.""" + """The identity the task's phases run as, if either names one.""" return self.agent.user if self.agent.user is not None else self.verifier.user @@ -191,6 +201,8 @@ def workspace_policy(task_dir: Path) -> dict[str, Any]: "agent_env": dict(config.agent.env), "workdir": config.environment.workdir or None, "user": config.user, + "agent_user": config.phase_user("agent"), + "verifier_user": config.phase_user("verifier"), } diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index 0686541c5..206dd1b36 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -11,6 +11,7 @@ import os import re import subprocess +from types import SimpleNamespace from typing import TYPE_CHECKING import pytest @@ -272,6 +273,10 @@ def test_declared_workspace_policy_is_translated(tmp_path) -> None: "agent_env": {}, "workdir": "/srv/app", "user": None, + # Per phase, because Harbor's are: a task may restrict the agent and + # still verify as root. + "agent_user": None, + "verifier_user": None, } @@ -352,23 +357,41 @@ async def run_tests(): assert grade["score"] == 0.0 -async def test_source_dockerfile_user_is_restored_after_the_layer(tmp_path) -> None: - # The layer installs as root; an image whose own Dockerfile ends on a - # non-root USER must get that identity back, or adaptation would grant - # root where Harbor withheld it. +async def test_a_declared_identity_governs_the_phases_not_the_harness(tmp_path) -> None: + # Harbor runs the agent and the verifier as the identity the task declares + # for each; its harness is not subject to either. Serving from inside the + # image, a USER directive would demote the harness too, leaving it unable + # to create /tests at the filesystem root or its own state under /hud. task = _write_harbor_task(tmp_path, "task-a") (task / "environment" / "Dockerfile").write_text( "FROM python:3.12-slim\nRUN useradd -m agent\nUSER agent\n", encoding="utf-8" ) await harbor.adapt(tmp_path, build=False) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) + layer = (context / "Dockerfile").read_text(encoding="utf-8").split("adaptation layer", 1)[1] - dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") + assert "USER root" in layer # installs, and goes on serving, as root + assert "USER agent" not in layer + assert "mkdir -p /tests" not in layer # root makes them when grading needs them + # The image's own declaration is not lost: it is kept for serve time, + # where the build context no longer exists. + assert (context / "_hud" / "image-user").read_text(encoding="utf-8").strip() == "agent" + + +def test_each_phase_takes_the_identity_declared_for_it(tmp_path, monkeypatch) -> None: + # A task may hand the agent a restricted account and still verify as root. + from integrations.harbor import _adapt + + task = _write_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text( + 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[agent]\nuser = "agent"\n', + encoding="utf-8", + ) + monkeypatch.setattr(_adapt, "_image_user", lambda _task: None) + monkeypatch.setattr(_adapt.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=1000)) - tail = dockerfile[dockerfile.index("HUD adaptation layer") :] - assert "USER root" in tail - assert "RUN chown -R agent /tests /logs" in tail - assert tail.rindex("USER agent") > tail.index("USER root") + assert _adapt.phase_uid(task, "agent") == 1000 + assert _adapt.phase_uid(task, "verifier") is None # root, as declared by omission def test_build_stage_entrypoint_does_not_refuse(tmp_path) -> None: From fc8ae82365ff2d49a199a4015f94d7dd2fe57104 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:42:21 -0700 Subject: [PATCH 13/33] fix(integrations): move the harness off the root of the task's filesystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adaptation layer kept itself at /hud, masked with a tmpfs. That hid what was in it and left the directory: a name at the root of the filesystem that the task's own image never had, plus the same name in the sandbox argv that pid 1 reports. Nothing of ours was readable through it, so this is not about confidentiality — it is the last thing in the agent's namespace identifying what is running it. Keep it under /media instead and rebuild that parent for sessions: a fresh tmpfs, then every sibling bound back as it was, so the entry is absent rather than present-and-empty. Only --tmpfs and --bind, so it holds on the bubblewrap that ships with bullseye, and writes elsewhere — including new directories at the filesystem root, which a task like configure-git-webserver needs — still land in the real filesystem for the verifier. What is given up is entries created directly in that one parent during a rollout, which is why it is a parent nothing writes to: /media is for removable media, where /srv and /opt are where service data and add-on software legitimately go. The path is a placeholder in the templates now rather than a literal, so it is stated once. An agent session sees a root with no /hud, no /tests and no /logs — the image's own filesystem, plus the sandbox it is running in. --- integrations/harbor/_adapt.py | 102 +++++++++++++-------- integrations/harbor/_load.py | 13 ++- integrations/harbor/tests/test_contract.py | 42 +++++---- 3 files changed, 97 insertions(+), 60 deletions(-) diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 486638be0..3a1f2d80a 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -13,16 +13,16 @@ "serve", "harbor:environment", "--arg", - "ref=/hud/tasks", + "ref=/media/hud/tasks", "--arg", "name=", ] Everything it needs is container-local: the workspace is the image's working directory, and grading runs each task's ``tests/test.sh`` in place. The -sandbox exists to keep the graded material — the baked tasks under ``/hud`` -and the verifier's verdict — outside the agent's namespace, and to sever the -network when a task declares no-network:: +sandbox exists to keep the graded material — the baked tasks under the +harness's own tree, and the verifier's verdict — outside the agent's +namespace, and to sever the network when a task declares no-network:: images = await harbor.adapt("./harbor_tasks", push="registry.io/acme") taskset = harbor.load("./harbor_tasks", images=images) @@ -57,6 +57,7 @@ from ._load import ( DEFAULT_VERIFIER_TIMEOUT, + HUD_ROOT, TaskConfig, final_stage, grouped, @@ -75,12 +76,6 @@ VERIFIER_LOGS = LOGS / "verifier" TESTS = Path("/tests") -#: Where the adaptation layer keeps itself, and the one path agent sessions -#: never see. The session's own key material belongs here rather than in a -#: temp dir: /tmp is the task's, so keys left there are both readable by the -#: graded party and a signpost saying what is running it. -HUD_ROOT = Path("/hud") - #: Task config is untrusted: these bound what may reach a generated directive. _DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*") @@ -103,26 +98,26 @@ _INSTALL_SH = """\ #!/bin/sh -# Install a self-contained hud venv under /hud: bootstrap uv (which brings its +# Install a self-contained hud venv under __HUD_ROOT__: bootstrap uv (which brings its # own managed Python), never touching the image's Python or site-packages. # Needs network and one of: uv / curl / wget / pip (+ apt-get/apk for bare # images with no downloader). set -eu -# Everything HUD installs goes under /hud — the uv binary, its managed -# interpreter, that interpreter's shims — because /hud is the one path the -# workspace masks from the graded party. Left at uv's defaults this state lands +# Everything HUD installs goes under __HUD_ROOT__ — the uv binary, its managed +# interpreter, that interpreter's shims — because that tree is the one the +# workspace keeps out of the graded party's namespace. At uv's defaults this lands # in the invoking user's home, which is *inside* the task's filesystem: the # agent then finds a HUD runtime the task never declared, and shims pointing # into the mask that resolve to nothing. Containing it is what makes the single -# /hud mask sufficient, rather than something to be cleaned up after the fact. +# single hiding of that tree sufficient, rather than state to clean up after. # The cache is the same argument, minus the keeping: nothing reads it after this # script, so it is never written. -export UV_INSTALL_DIR=/hud/bin \\ - UV_PYTHON_INSTALL_DIR=/hud/python \\ - UV_PYTHON_BIN_DIR=/hud/bin \\ +export UV_INSTALL_DIR=__HUD_ROOT__/bin \\ + UV_PYTHON_INSTALL_DIR=__HUD_ROOT__/python \\ + UV_PYTHON_BIN_DIR=__HUD_ROOT__/bin \\ UV_NO_CACHE=1 \\ - XDG_CONFIG_HOME=/hud/config -export PATH="/hud/bin:$PATH" + XDG_CONFIG_HOME=__HUD_ROOT__/config +export PATH="__HUD_ROOT__/bin:$PATH" command -v uv >/dev/null 2>&1 || { { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \\ || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \\ @@ -141,8 +136,8 @@ || apk add --no-cache bubblewrap \ || echo "warning: bubblewrap unavailable; tasks declaring isolation will refuse to serve" uv python install __PYTHON__ -uv venv /hud/venv --python __PYTHON__ -uv pip install --python /hud/venv/bin/python __HUD_REQUIREMENT__ +uv venv __HUD_ROOT__/venv --python __PYTHON__ +uv pip install --python __HUD_ROOT__/venv/bin/python __HUD_REQUIREMENT__ # ─── the agent's toolchain ─── # Harbor installs the agent *into* the container, and that step provisions what @@ -180,14 +175,14 @@ # ─── HUD adaptation layer: serve the control channel from inside ─── # (generated by harbor.adapt) # Installed as root: the base stage may end on a non-root USER that could not -# write /hud. The task's own declared user, if any, is restored below. +# write __HUD_ROOT__. The task's own declared user, if any, is restored below. USER root -COPY _hud /hud -RUN sh /hud/install.sh +COPY _hud __HUD_ROOT__ +RUN sh __HUD_ROOT__/install.sh # This integration ships outside the hud distribution, so it comes from the # build context rather than the index — which also means the code serving the # image is the revision that adapted it, not whatever the index resolves to. -COPY _hud_harbor /hud/venv/lib/python__PYTHON__/site-packages/harbor +COPY _hud_harbor __HUD_ROOT__/venv/lib/python__PYTHON__/site-packages/harbor # The CLI's update check has no user to prompt here, and its cache lives in the # invoking user's home — inside the task's filesystem. A container is fresh # every rollout, so the cache is never warm: left on, it calls PyPI on the @@ -195,8 +190,8 @@ ENV HUD_SKIP_VERSION_CHECK=1 EXPOSE 8765 __DECLARED__ENTRYPOINT [] -CMD ["/hud/venv/bin/hud", "serve", \ - "harbor:environment", "--arg", "ref=/hud/tasks", \ +CMD ["__HUD_ROOT__/venv/bin/hud", "serve", \ + "harbor:environment", "--arg", "ref=__HUD_ROOT__/tasks", \ "--arg", "name=__ENV_NAME__", "--host", "0.0.0.0", "--port", "8765"] """ @@ -383,7 +378,7 @@ def _write_context( wheel = Path(hud_requirement) if wheel.suffix == ".whl" and wheel.is_file(): shutil.copy2(wheel, hud_dir / wheel.name) - requirement = f"/hud/{wheel.name}" + requirement = str(HUD_ROOT / wheel.name) shutil.copytree( Path(__file__).parent, context / "_hud_harbor", @@ -397,9 +392,9 @@ def _write_context( _copy_task_content(task_dir / "tests", target / "tests") _write( hud_dir / "install.sh", - _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)).replace( - "__PYTHON__", SERVING_PYTHON - ), + _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)) + .replace("__PYTHON__", SERVING_PYTHON) + .replace("__HUD_ROOT__", str(HUD_ROOT)), ) # The image's own USER, kept for serve time: the build context is gone by # then, and a task that declares no user of its own still runs its phases @@ -407,7 +402,8 @@ def _write_context( if (image_user := final_stage(dockerfile).user) is not None: _write(hud_dir / "image-user", f"{image_user}\n") layer = ( - _LAYER.replace("__ENV_NAME__", env_name) + _LAYER.replace("__HUD_ROOT__", str(HUD_ROOT)) + .replace("__ENV_NAME__", env_name) .replace("__PYTHON__", SERVING_PYTHON) .replace( # One env serves one policy, so the group's tasks agree on these. @@ -453,7 +449,33 @@ def docker_runtime(**kwargs: Any) -> DockerRuntime: # ─── what an adapted image serves, from inside the container ──────────── -def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> Environment: +def _harness_out_of_view() -> tuple[Mount, ...]: + """Mounts that leave the harness's tree absent from a session's namespace. + + A tmpfs over the tree itself would hide what is in it and still leave the + directory — and a directory named after the harness, at a path the task's + own image never had, is the one thing left telling the graded party what + is running it. Rebuilding the parent removes the entry instead: a fresh + tmpfs, then every sibling bound back as it was. Only ``--tmpfs`` and + ``--bind``, so it holds on the bubblewrap that ships with bullseye. + + What the sandbox loses is entries created *directly* in that parent during + the rollout, which is why it is one nothing writes to: writes anywhere + else, including new directories at the filesystem root, still land in the + real filesystem for the verifier to grade. + """ + parent = HUD_ROOT.parent + mounts = [Mount("tmpfs", dst=str(parent))] + if parent.is_dir(): + mounts += [ + Mount("rw", src=str(sibling), dst=str(sibling)) + for sibling in sorted(parent.iterdir()) + if sibling != HUD_ROOT + ] + return tuple(mounts) + + +def environment(ref: str | Path = HUD_ROOT / "tasks", *, name: str | None = None) -> Environment: """The live env serving the task dirs under *ref* — the contract verb. Harbor environments are container filesystems, so this constructor is @@ -501,12 +523,12 @@ def environment(ref: str | Path = "/hud/tasks", *, name: str | None = None) -> E ), # Masks go in ``mounts``, which bwrap applies *after* the workspace # bind — as system mounts they would be re-covered when the guest - # path is ``/`` (an image with no WORKDIR). Only the harness's own - # tree is masked: it has to exist for the whole rollout, since it is - # what is serving. ``/tests`` and the verdict are handled by not - # existing during the agent phase at all (:func:`_hide_grading_dirs`), - # which is what Harbor does — it uploads the verifier when it runs it. - mounts=(Mount("tmpfs", dst=str(HUD_ROOT)),), + # path is ``/`` (an image with no WORKDIR). ``/tests`` and the verdict + # are handled by not existing during the agent phase at all + # (:func:`_hide_grading_dirs`), which is what Harbor does; the + # harness's own tree has to exist throughout, being what serves, so it + # is hidden by rebuilding its parent instead. + mounts=_harness_out_of_view(), credentials_dir=HUD_ROOT / "session-keys", # The agent's declared identity, not the harness's: the serving # process stays root so it can place and remove the grading diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py index 8433465f5..84e60f322 100644 --- a/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -20,6 +20,13 @@ LOGGER = logging.getLogger(__name__) +#: Where an adapted image keeps the harness: its venv, the baked tasks, the +#: session keys, the grading material. Nested rather than at the root, and +#: rebuilt out of the agent's namespace by :func:`~harbor.environment`, +#: so what the graded party sees is the empty ``/media`` any container has — +#: not a directory named after the thing evaluating it. +HUD_ROOT = Path("/media/hud") + DEFAULT_VERIFIER_TIMEOUT = 600.0 @@ -266,10 +273,10 @@ def unsupported_features(task_dir: Path) -> list[str]: # both phases run as it; two *different* identities are not. reasons.append("agent.user and verifier.user differ (the image has one USER)") workdir = environment.workdir or _final_stage_workdir(task_dir) - if workdir and (workdir == "/hud" or workdir.startswith("/hud/")): - # The adaptation layer owns /hud inside the image and hides it from + if workdir and Path(workdir).is_relative_to(HUD_ROOT): + # The adaptation layer owns that tree inside the image and hides it from # agent sessions; a task working there would find it empty. - reasons.append(f"working directory {workdir!r} is inside /hud (reserved by adaptation)") + reasons.append(f"working directory {workdir!r} is inside {HUD_ROOT} (reserved)") if environment.docker_image and not (task_dir / "environment" / "Dockerfile").is_file(): reasons.append( "prebuilt docker_image environments (adapt builds from environment/Dockerfile)" diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index 206dd1b36..953ec9d95 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -72,13 +72,12 @@ async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None: assert (task_dir / "tests" / "test.sh").is_file() -async def test_the_layer_keeps_its_own_state_under_the_mask(tmp_path) -> None: - # /hud is the path the workspace masks from the graded party, so everything - # the layer installs belongs there: uv's binary, its managed interpreter and - # that interpreter's shims, its cache and config. At uv's defaults these - # land in the invoking user's home — inside the task's own filesystem, where - # the agent finds a HUD runtime the task never declared, and shims into the - # mask that resolve to nothing. +async def test_the_layer_keeps_its_own_state_out_of_the_task(tmp_path) -> None: + # Everything the layer installs belongs under the harness's own tree: uv's + # binary, its managed interpreter and that interpreter's shims, its cache + # and config. At uv's defaults these land in the invoking user's home — + # inside the task's own filesystem, where the agent finds a HUD runtime the + # task never declared, and shims into a hidden tree resolving to nothing. _write_harbor_task(tmp_path, "task-a") await harbor.adapt(tmp_path, build=False) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) @@ -86,7 +85,9 @@ async def test_the_layer_keeps_its_own_state_under_the_mask(tmp_path) -> None: directories = dict(re.findall(r"\b(UV_\w*(?:DIR|HOME)|XDG_CONFIG_HOME)=(\S+)", script)) assert directories, "the layer pins no uv directories" - assert all(path.startswith("/hud/") for path in directories.values()), directories + assert all(path.startswith(f"{harbor_load.HUD_ROOT}/") for path in directories.values()), ( + directories + ) # Nothing is written relative to the image's home, so there is no scattered # state to clean up afterwards. assert "$HOME" not in script @@ -153,8 +154,9 @@ def test_adapt_images_stamp_rows_when_the_caller_passes_them(tmp_path) -> None: async def test_environment_serves_the_baked_tasks(tmp_path, monkeypatch) -> None: - # The constructor refuses to build an unsandboxed env (the /hud mask is - # an integrity property); tests run outside a container, so stub bwrap. + # The constructor refuses to build an unsandboxed env (hiding the + # harness's tree is an integrity property); tests run outside a + # container, so stub bwrap. monkeypatch.setattr("hud.environment.workspace.usable_bwrap", lambda: "/usr/bin/true") _write_harbor_task(tmp_path, "task-a") _write_harbor_task(tmp_path, "task-b") @@ -226,8 +228,9 @@ def test_load_carries_metadata_as_columns(tmp_path) -> None: async def test_served_templates_use_the_declared_description(tmp_path, monkeypatch) -> None: - # The constructor refuses to build an unsandboxed env (the /hud mask is - # an integrity property); tests run outside a container, so stub bwrap. + # The constructor refuses to build an unsandboxed env (hiding the + # harness's tree is an integrity property); tests run outside a + # container, so stub bwrap. monkeypatch.setattr("hud.environment.workspace.usable_bwrap", lambda: "/usr/bin/true") task = _write_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text( @@ -361,7 +364,8 @@ async def test_a_declared_identity_governs_the_phases_not_the_harness(tmp_path) # Harbor runs the agent and the verifier as the identity the task declares # for each; its harness is not subject to either. Serving from inside the # image, a USER directive would demote the harness too, leaving it unable - # to create /tests at the filesystem root or its own state under /hud. + # to create /tests at the filesystem root or its own state under the + # harness tree. task = _write_harbor_task(tmp_path, "task-a") (task / "environment" / "Dockerfile").write_text( "FROM python:3.12-slim\nRUN useradd -m agent\nUSER agent\n", encoding="utf-8" @@ -515,7 +519,11 @@ def record(*args, **kwargs): (workspace,) = built masked = [m.dst for m in workspace.mounts] - assert "/hud" in masked + # The harness's tree is hidden by rebuilding its parent, so the entry is + # absent rather than present-and-empty: a directory named after the + # harness is the tell, not what is inside it. + assert str(harbor_load.HUD_ROOT.parent) in masked + assert str(harbor_load.HUD_ROOT) not in masked # The assertions and the verdict are kept from the agent phase by not # existing during it, not by masking: an empty directory the base image # does not have is itself a signal. @@ -773,11 +781,11 @@ def test_image_tags_do_not_depend_on_host_state(tmp_path) -> None: def test_a_workdir_inside_the_reserved_path_is_refused(tmp_path) -> None: - # /hud belongs to the adaptation layer and is hidden from agent sessions; + # That tree belongs to the adaptation layer and is hidden from sessions; # a task working there would find it empty. task = _write_harbor_task(tmp_path, "task-a") (task / "environment" / "Dockerfile").write_text( - "FROM python:3.12-slim\nWORKDIR /hud/app\n", encoding="utf-8" + f"FROM python:3.12-slim\nWORKDIR {harbor_load.HUD_ROOT}/app\n", encoding="utf-8" ) - assert "reserved by adaptation" in " ".join(harbor_load.unsupported_features(task)) + assert "reserved" in " ".join(harbor_load.unsupported_features(task)) From b05bd446a3a9269339b8f54f03b09998aa8dc6f0 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:05:18 -0700 Subject: [PATCH 14/33] fix(integrations): map ids for the verifier that declares no-network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verifier is sandboxed for one reason — a task declaring no-network needs a network namespace, and bwrap can only make one inside a user namespace. That namespace maps a single id, so the verifier could not chown: not untar with ownership, not run apt, which is most of what Harbor's verifiers do before they assert anything. It failed with EINVAL and graded 0.0, indistinguishable from a task the agent got wrong. Hold it at creation and map it, exactly as an agent session's sandbox is now. The handshake moves to the workspace as install_identity_map, since both sides of it were the same fifteen lines. Verified with a verifier that chowns and probes the network: chown=yes, network=severed. Fixing the ids does not hand back the network the task ruled out. --- hud/environment/workspace.py | 30 ++++++---- integrations/harbor/_adapt.py | 100 ++++++++++++++++++++++------------ 2 files changed, 86 insertions(+), 44 deletions(-) diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index bc0ed9772..00aff3b9c 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -190,6 +190,22 @@ def _env_argv(env: Mapping[str, str]) -> list[str]: return [env_bin, "-i", *(f"{k}={v}" for k, v in env.items())] +async def install_identity_map(info_read: int, block_write: int) -> int: + """Map ids into a bwrap held at ``--userns-block-fd``, and release it. + + The counterpart to spawning with ``--info-fd``/``--userns-block-fd``: + bwrap reports the pid of the namespace it made and waits, this side says + who its ids are, and only then does anything run in it. Returns that pid. + """ + loop = asyncio.get_running_loop() + raw = await asyncio.wait_for(loop.run_in_executor(None, os.read, info_read, 4096), 30.0) + pid = int(json.loads(raw)["child-pid"]) if raw else 0 + if pid: + _map_identities(pid) + os.write(block_write, b"\n") + return pid + + def _map_identities(pid: int) -> None: """Give the sandbox the whole id space, not just the id that created it. @@ -785,23 +801,17 @@ async def _start_sandbox(self) -> int: os.close(write_fd) os.close(block_read) write_fd = block_read = -1 - loop = asyncio.get_running_loop() - raw = await asyncio.wait_for(loop.run_in_executor(None, os.read, read_fd, 4096), 30.0) - if raw: - # The sandbox is held at its own creation until this is - # written: nothing runs in it, and nothing can join it, before - # it knows who its ids are. - _map_identities(int(json.loads(raw)["child-pid"])) - os.write(block_write, b"\n") + # The sandbox is held at its own creation until its ids are + # mapped: nothing runs in it, and nothing joins it, before then. + pid = await install_identity_map(read_fd, block_write) finally: os.close(read_fd) os.close(block_write) for stray in (write_fd, block_read): if stray != -1: os.close(stray) - if not raw: + if not pid: raise RuntimeError(f"the sandbox holder did not start: {await self._sandbox_error()}") - pid = int(json.loads(raw)["child-pid"]) assert self._sandbox is not None and self._sandbox.stdout is not None try: signal = await asyncio.wait_for(self._sandbox.stdout.readline(), 30.0) diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 3a1f2d80a..b16e82893 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -50,7 +50,7 @@ from typing import Any from hud.environment import Environment, Mount -from hud.environment.workspace import Workspace, usable_bwrap +from hud.environment.workspace import Workspace, install_identity_map, usable_bwrap from hud.eval import DockerRuntime from hud.utils.docker import docker as _docker from hud.utils.process import ProcessGroup, create_process_group_exec @@ -661,41 +661,73 @@ async def _grade(task_dir: Path, workdir: Path, answer: Any) -> dict[str, Any]: "--clear-groups", *argv, ] - if not config.network("verifier"): - # The verifier runs outside the agent sandbox, so its declared - # isolation needs its own network namespace. - bwrap = usable_bwrap() - if bwrap is None: - raise RuntimeError( - "the verifier declares no-network but bwrap cannot sandbox here; " - "refusing to grade with network access the task ruled out" - ) - # Mirror a session's namespace shape: binding the real root inside a - # user namespace leaves device nodes unwritable (test.sh redirecting - # to /dev/null would fail), so /dev and /proc are fresh. - argv = [ - bwrap, - "--unshare-user-try", - "--bind", - "/", - "/", - "--dev", - "/dev", - "--proc", - "/proc", - "--unshare-net", - "--", - *argv, - ] + severed = not config.network("verifier") + if severed and (bwrap := usable_bwrap()) is None: + raise RuntimeError( + "the verifier declares no-network but bwrap cannot sandbox here; " + "refusing to grade with network access the task ruled out" + ) async def run_tests() -> ProcessGroup: - return await create_process_group_exec( - *argv, - cwd=workdir, - env={**os.environ, **verifier_env, **config.verifier.env}, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) + spawn = argv + environ = {**os.environ, **verifier_env, **config.verifier.env} + if not severed: + return await create_process_group_exec( + *spawn, + cwd=workdir, + env=environ, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + # A network namespace of its own is the only reason the verifier is + # sandboxed at all, and bwrap can only make one inside a user + # namespace. That namespace maps a single id unless this side says + # otherwise — and a verifier that cannot chown is one that cannot + # untar, or run apt, which is most of what Harbor's verifiers do + # before they assert anything. So it is held at creation and mapped, + # exactly as an agent session's sandbox is. + info_read, info_write = os.pipe() + block_read, block_write = os.pipe() + try: + os.set_inheritable(info_write, True) + os.set_inheritable(block_read, True) + # /dev and /proc are fresh: binding the real root inside a user + # namespace leaves device nodes unwritable, and a test.sh + # redirecting to /dev/null would fail on that alone. + group = await create_process_group_exec( + bwrap, + "--unshare-user", + "--info-fd", + str(info_write), + "--userns-block-fd", + str(block_read), + "--bind", + "/", + "/", + "--dev", + "/dev", + "--proc", + "/proc", + "--unshare-net", + "--", + *spawn, + cwd=workdir, + env=environ, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + pass_fds=(info_write, block_read), + ) + os.close(info_write) + os.close(block_read) + info_write = block_read = -1 + await install_identity_map(info_read, block_write) + return group + finally: + os.close(info_read) + os.close(block_write) + for stray in (info_write, block_read): + if stray != -1: + os.close(stray) return await _grade_with_verifier(config, logs, answer, run_tests) From 812ae12c7e5f1612af20a05e3fac7dcbffd4a658 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:03:51 -0700 Subject: [PATCH 15/33] feat(environment): give the workspace its own network, and a policy on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workspace shared the substrate's network, so a session could address anything else listening on it — the control channel included. That is how a graded party could call tasks.grade, read the verifier's own output, and take the session key out of hello. Reachability was the whole of it: nothing was wrong with the channel, it was simply on the same network as the thing being graded. So give the workspace a network of its own and one way out of it. The proxy listens on a unix socket, which makes reaching it a question of the filesystem rather than the network; a bridge runs in the workspace's network namespace while keeping this one's mount namespace, so it can see a socket the workspace cannot, and offers it as an ordinary proxy port on the workspace's loopback. Nothing of ours is bound inside, and the substrate is not addressable from in there — the agent's namespace ends up with exactly one listener, which is the proxy. Every connection passing one process is also what makes a policy possible, so Harbor's network_mode stops being refused: an allowlist is the hosts the task named, no-network is none of them, and public leaves the network as it was. The verifier's allowlist is still refused, because it runs on the substrate's network with nothing between it and the hosts it dials. Parsing is the standard library's. A hand-rolled proxy got keep-alive and body framing wrong in the way that matters here: apt failed halfway through an index, which reads as a broken task rather than a broken harness. --- hud/environment/egress.py | 270 +++++++++++++++++++++ hud/environment/tests/test_workspace.py | 33 +++ hud/environment/workspace.py | 43 +++- integrations/harbor/_adapt.py | 5 + integrations/harbor/_load.py | 38 ++- integrations/harbor/tests/test_contract.py | 16 +- 6 files changed, 396 insertions(+), 9 deletions(-) create mode 100644 hud/environment/egress.py diff --git a/hud/environment/egress.py b/hud/environment/egress.py new file mode 100644 index 000000000..ba0aedeed --- /dev/null +++ b/hud/environment/egress.py @@ -0,0 +1,270 @@ +"""The one way out of a bounded workspace, and the policy on it. + +A workspace with its own network namespace has no route anywhere — not to the +internet, and not to whatever else the substrate is running, including the +control channel that grades it. Egress is given back deliberately, through a +proxy that sees every connection and applies the task's declared policy. + +The proxy listens on a unix socket, so reaching it is a question of the +filesystem rather than the network: a bridge runs in the workspace's *network* +namespace while keeping the substrate's *mount* namespace, so it can see the +socket the workspace itself cannot, and offers it as an ordinary proxy port on +the workspace's loopback. Nothing is bound into the workspace, and nothing in +it can address the substrate. + +Request parsing is the standard library's. A hand-rolled request-line parser +gets keep-alive, chunked bodies and header framing wrong in ways that surface +as a package manager failing halfway through an index rather than as an +obvious error. +""" + +from __future__ import annotations + +import contextlib +import http.client +import logging +import os +import select +import shutil +import socket +import socketserver +import subprocess +import sys +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Collection + from pathlib import Path + +LOGGER = logging.getLogger("hud.environment.egress") + +#: In an allowlist, the entry that permits everything. +ANY_HOST = "*" + +#: Headers that belong to one hop and must not be forwarded to the next. +_HOP_BY_HOP = frozenset( + {"connection", "proxy-connection", "keep-alive", "te", "trailers", "upgrade"} +) + +#: The proxy port offered on the workspace's loopback. 3128 is unremarkable — +#: an egress proxy is ordinary infrastructure, unlike a control channel. +BRIDGE_PORT = 3128 + +_BRIDGE = """ +import asyncio, sys + +async def splice(reader, writer): + try: + while chunk := await reader.read(65536): + writer.write(chunk) + await writer.drain() + except Exception: + pass + finally: + try: + writer.close() + except Exception: + pass + +async def bridged(reader, writer): + up_reader, up_writer = await asyncio.open_unix_connection(sys.argv[1]) + await asyncio.gather(splice(reader, up_writer), splice(up_reader, writer)) + +async def main(): + server = await asyncio.start_server(bridged, "127.0.0.1", int(sys.argv[2])) + async with server: + await server.serve_forever() + +asyncio.run(main()) +""" + + +def permitted(host: str | None, allowed: Collection[str]) -> bool: + """Whether *host* is in *allowed*, by exact match or as a subdomain.""" + if not host: + return False + if ANY_HOST in allowed: + return True + return any(host == entry or host.endswith(f".{entry}") for entry in allowed) + + +class _Proxy(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + allowed: Collection[str] = () + + def log_message(self, *_: object) -> None: + """The workspace's traffic is not the substrate's log.""" + + def _deny(self) -> None: + # Loud and diagnosable from inside the workspace: a host held back by + # policy should not look like a network that is merely broken. + self.send_response(403) + self.send_header("X-Proxy-Error", "blocked-by-allowlist") + self.send_header("Content-Length", "0") + self.end_headers() + + def do_CONNECT(self) -> None: + host, _, port = self.path.rpartition(":") + if not permitted(host, self.allowed): + self._deny() + return + try: + upstream = socket.create_connection((host, int(port or 443)), timeout=15) + except (OSError, ValueError): + self.send_error(502) + return + self.send_response(200, "Connection established") + self.end_headers() + client = self.connection + with upstream: + while True: + ready, _, _ = select.select([client, upstream], [], [], 300) + if not ready: + return + for source in ready: + target = upstream if source is client else client + try: + data = source.recv(65536) + if not data: + return + target.sendall(data) + except OSError: + return + + def _forward(self) -> None: + parts = urllib.parse.urlsplit(self.path) + if not permitted(parts.hostname, self.allowed): + self._deny() + return + body = None + if length := self.headers.get("Content-Length"): + body = self.rfile.read(int(length)) + headers = {k: v for k, v in self.headers.items() if k.lower() not in _HOP_BY_HOP} + # Rebuilt from the parsed components rather than forwarded raw: the + # policy was applied to *this* hostname, and the request that goes out + # must be the one it was applied to. + path = urllib.parse.urlunsplit(("", "", parts.path or "/", parts.query, "")) + connection = http.client.HTTPConnection(parts.hostname or "", parts.port or 80, timeout=60) + try: + connection.request(self.command, path, body=body, headers=headers) + response = connection.getresponse() + self.send_response(response.status, response.reason) + length = response.getheader("Content-Length") + for key, value in response.getheaders(): + if key.lower() not in _HOP_BY_HOP and key.lower() != "content-length": + self.send_header(key, value) + if length is not None: + self.send_header("Content-Length", length) + else: + # Nothing upstream framed the body, so the close delimits it. + self.send_header("Connection", "close") + self.close_connection = True + self.end_headers() + shutil.copyfileobj(response, self.wfile) + except (OSError, http.client.HTTPException): + self.close_connection = True + finally: + connection.close() + + do_GET = _forward + do_HEAD = _forward + do_POST = _forward + do_PUT = _forward + do_DELETE = _forward + do_PATCH = _forward + do_OPTIONS = _forward + + +class _UnixProxyServer(socketserver.ThreadingUnixStreamServer): + daemon_threads = True + + def get_request(self) -> tuple[socket.socket, tuple[str, int]]: + # A unix peer has no address; the handler wants one to log. + request, _ = super().get_request() + return request, ("workspace", 0) + + +class Egress: + """A workspace's route out, and the policy applied to it. + + ``allowed`` is the set of hosts a session may reach — ``{ANY_HOST}`` for + all of them. An empty set is a workspace that can reach nothing, which is + also what not starting one at all means. + """ + + def __init__(self, socket_path: Path | str, allowed: Collection[str]) -> None: + self.socket_path = str(socket_path) + self.allowed = frozenset(allowed) + self._server: _UnixProxyServer | None = None + self._thread: threading.Thread | None = None + self._bridge: subprocess.Popen[bytes] | None = None + + def start(self) -> None: + """Serve the policy on the unix socket. Idempotent.""" + if self._server is not None: + return + with contextlib.suppress(FileNotFoundError): + os.unlink(self.socket_path) + os.makedirs(os.path.dirname(self.socket_path) or ".", exist_ok=True) + handler = type("_ScopedProxy", (_Proxy,), {"allowed": self.allowed}) + self._server = _UnixProxyServer(self.socket_path, handler) + os.chmod(self.socket_path, 0o600) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: + """Offer the proxy on the loopback of *pid*'s network namespace. + + The bridge joins that namespace and nothing else, so it keeps this + filesystem — which is how it reaches a socket the workspace cannot. + """ + nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" + self._bridge = subprocess.Popen( + [ + nsenter, + "--target", + str(pid), + "--net", + "--user", + "--preserve-credentials", + "--", + sys.executable, + "-c", + _BRIDGE, + self.socket_path, + str(port), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def environment(self, port: int = BRIDGE_PORT) -> dict[str, str]: + """Proxy variables for a session, in the spellings clients read.""" + url = f"http://127.0.0.1:{port}" + return { + "http_proxy": url, + "https_proxy": url, + "HTTP_PROXY": url, + "HTTPS_PROXY": url, + "no_proxy": "127.0.0.1,localhost", + "NO_PROXY": "127.0.0.1,localhost", + } + + def stop(self) -> None: + """Take the route away.""" + if self._bridge is not None: + self._bridge.kill() + self._bridge = None + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + self._thread = None + with contextlib.suppress(FileNotFoundError): + os.unlink(self.socket_path) + + +__all__ = ["ANY_HOST", "BRIDGE_PORT", "Egress", "permitted"] diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index ae9ccbec7..f6908e760 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -363,6 +363,39 @@ def test_the_sandbox_reports_readiness_before_sessions_join_it( assert argv[-1] == "echo ready" +def test_making_a_network_and_joining_it_are_the_same_question( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A sandbox that owns a network must be entered through it. Answered + differently in the two places, sessions run on the substrate's network — + the one the policy exists to keep them off — while the sandbox sits in an + empty namespace nothing ever enters.""" + for allowed, network, owns in ( + ({"example.com"}, True, True), # a policy needs a network to apply to + (set(), True, True), # declared unreachable + (None, False, True), # no-network, however it was spelled + (None, True, False), # the substrate's network, as before + ): + ws = Workspace(tmp_path / "root", network=network, allowed_hosts=allowed) + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + + assert ws.owns_netns is owns + assert ("--unshare-net" in ws.bwrap_argv(["true"])) is owns + assert ("--net" in ws.enter_argv(7, "true")) is owns + + +def test_a_host_is_permitted_by_name_or_as_a_subdomain() -> None: + from hud.environment.egress import ANY_HOST, permitted + + assert permitted("pypi.org", {"pypi.org"}) + assert permitted("files.pypi.org", {"pypi.org"}) # a subdomain of what was named + assert not permitted("notpypi.org", {"pypi.org"}) # not a subdomain, a different host + assert not permitted("pypi.org.evil.com", {"pypi.org"}) + assert not permitted("anything", set()) # declared unreachable + assert permitted("anything", {ANY_HOST}) + assert not permitted(None, {ANY_HOST}) + + def test_shell_uid_wraps_sessions_in_setpriv( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 00aff3b9c..03656957e 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -19,6 +19,7 @@ import asyncssh +from hud.environment.egress import Egress from hud.utils.process import create_process_group_exec if sys.platform != "win32": # the pty a session runs on has no Windows analogue @@ -27,7 +28,7 @@ import termios if TYPE_CHECKING: - from collections.abc import Mapping, Sequence + from collections.abc import Collection, Mapping, Sequence from hud.capabilities import Capability @@ -325,6 +326,7 @@ def __init__( # bwrap configuration mounts: Sequence[Mount] = (), network: bool = False, + allowed_hosts: Collection[str] | None = None, env: Mapping[str, str] | None = None, system_mounts: Sequence[Mount] | None = None, guest_path: str = "/workspace", @@ -353,6 +355,15 @@ def __init__( # bwrap state self.mounts: tuple[Mount, ...] = tuple(mounts) self.network = network + #: Which hosts a session may reach. ``None`` leaves the network as the + #: substrate's — sessions share it, and so can reach whatever else is + #: listening there, the control channel included. A set (``{ANY_HOST}`` + #: for everything) gives the workspace its own network namespace whose + #: only route out is the policy: nothing else on the substrate is + #: addressable from inside, and no session can reach a host the task + #: did not declare. + self.allowed_hosts = None if allowed_hosts is None else frozenset(allowed_hosts) + self._egress: Egress | None = None self.env: dict[str, str] = dict(env or {}) self._system_mounts: tuple[Mount, ...] = tuple( system_mounts if system_mounts is not None else DEFAULT_SYSTEM_MOUNTS, @@ -413,6 +424,17 @@ def __init__( # and two that each started a sandbox would not share one. self._sandbox_lock = asyncio.Lock() + @property + def owns_netns(self) -> bool: + """Whether the workspace has a network of its own. + + True when the task severed the network, and true when it declared what + may be reached — a policy needs somewhere to apply. False leaves + sessions on the substrate's network, where they can address whatever + else is listening on it. + """ + return not self.network or self.allowed_hosts is not None + def _setpriv(self) -> str | None: """Absolute path to ``setpriv``, resolved via the *server's* PATH. @@ -674,7 +696,7 @@ def bwrap_argv( "--unshare-uts", "--unshare-cgroup-try", ] - if not self.network: + if self.owns_netns: argv.append("--unshare-net") if info_fd is not None: argv.extend(["--info-fd", str(info_fd)]) @@ -722,7 +744,10 @@ def enter_argv( "--pid", "--uts", "--ipc", - *(() if self.network else ("--net",)), + # Joined exactly when the sandbox has a network of its own — + # otherwise a session would run on the substrate's, which is the + # network the workspace was given a policy to keep it off. + *(("--net",) if self.owns_netns else ()), f"--wd={cwd if cwd is not None else self._guest_path}", "--", ] @@ -738,9 +763,10 @@ def _full_env(self, env: Mapping[str, str] | None = None) -> dict[str, str]: Dropped sessions get the minimal one built for the wall; otherwise the serving process's environment carries through, less HUD's own. """ + proxy = self._egress.environment() if self._egress is not None else {} if self._drops_privileges(): - return {**(self._session_env() or {}), **(env or {})} - return {**_without_harness_config(os.environ), **self.env, **(env or {})} + return {**(self._session_env() or {}), **proxy, **(env or {})} + return {**_without_harness_config(os.environ), **proxy, **self.env, **(env or {})} def _drop_argv(self) -> list[str]: """The ``setpriv`` prefix that drops to ``shell_uid``, if it applies.""" @@ -822,6 +848,10 @@ async def _start_sandbox(self) -> int: await self.discard_sandbox() raise RuntimeError(f"the sandbox never became ready: {reason}") self._sandbox_init = pid + if self.allowed_hosts: + self._egress = Egress(self._credentials_dir() / "egress.sock", self.allowed_hosts) + self._egress.start() + self._egress.attach(pid) return pid async def _sandbox_error(self) -> str: @@ -842,6 +872,9 @@ async def discard_sandbox(self) -> None: every process the agent left behind goes with it — including ones it detached. A later session starts a fresh sandbox. """ + if self._egress is not None: + self._egress.stop() + self._egress = None sandbox, self._sandbox, self._sandbox_init = self._sandbox, None, None if sandbox is None or sandbox.returncode is not None: return diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index b16e82893..f10ae3616 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -542,6 +542,11 @@ def environment(ref: str | Path = HUD_ROOT / "tasks", *, name: str | None = None # session would otherwise be pointed at the harness's home. env={**({"HOME": home} if (home := phase_home(agent_uid)) else {}), **policy["agent_env"]}, network=policy["network"], + # What the task declared its agent may reach. Given a set, the + # workspace takes a network of its own whose only way out applies it — + # which is also what puts the substrate, and the channel grading this + # rollout, out of the agent's reach. + allowed_hosts=policy["allowed_hosts"], # Always: the sandbox is what keeps the baked tests and the serving # venv out of the graded party's reach — an unsandboxed fallback # would hand the agent the verifier's answers. diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py index 84e60f322..5e1f9e5e9 100644 --- a/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -38,6 +38,7 @@ class _Phase(BaseModel): timeout_sec: float | None = Field(default=None, gt=0) user: str | int | None = None network_mode: str | None = None + allowed_hosts: list[str] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) environment: dict[str, Any] | None = None environment_mode: str | None = None @@ -57,6 +58,7 @@ class _EnvironmentSection(BaseModel): gpu_types: list[str] = Field(default_factory=list) tpu: dict[str, Any] | None = None network_mode: str | None = None + allowed_hosts: list[str] = Field(default_factory=list) workdir: str | None = None env: dict[str, str] = Field(default_factory=dict) healthcheck: dict[str, Any] | None = None @@ -117,7 +119,7 @@ def phase(self, role: str) -> _Phase: return self.agent if role == "agent" else self.verifier def network(self, role: str) -> bool: - """Whether *role*'s processes may reach the network. + """Whether *role*'s processes may reach the network at all. Harbor declares isolation container-wide or per phase; either severs that phase, so the workspace and the verifier cannot disagree about @@ -125,6 +127,22 @@ def network(self, role: str) -> bool: """ return "no-network" not in (self.environment.network_mode, self.phase(role).network_mode) + def allowed_hosts(self, role: str) -> frozenset[str] | None: + """Which hosts *role* may reach, or None to leave its network alone. + + A phase declaring ``allowlist`` may reach the hosts it names and + nothing else — including nothing else on the substrate, which is where + the harness serving it lives. ``public`` names no hosts and so permits + all of them; ``no-network`` is the empty set, which permits none. + """ + mode = self.phase(role).network_mode or self.environment.network_mode or "public" + if mode == "no-network": + return frozenset() + if mode != "allowlist": + return None + declared = [*self.environment.allowed_hosts, *self.phase(role).allowed_hosts] + return frozenset(declared) + def phase_user(self, role: str) -> str | int | None: """The identity *role* runs as, if the task names one for it. @@ -208,6 +226,12 @@ def workspace_policy(task_dir: Path) -> dict[str, Any]: "agent_env": dict(config.agent.env), "workdir": config.environment.workdir or None, "user": config.user, + # Sorted rather than a set: the policy is hashed to key environments, + # so it has to serialize, and two tasks naming the same hosts in a + # different order declare the same thing. + "allowed_hosts": ( + None if (hosts := config.allowed_hosts("agent")) is None else sorted(hosts) + ), "agent_user": config.phase_user("agent"), "verifier_user": config.phase_user("verifier"), } @@ -262,8 +286,16 @@ def unsupported_features(task_dir: Path) -> list[str]: ("agent", agent.network_mode), ("verifier", verifier.network_mode), ): - if mode == "allowlist": - reasons.append(f"{role}.network_mode='allowlist' (per-host policy is not enforceable)") + if mode == "allowlist" and role in ("environment", "verifier"): + # The agent's allowlist is applied by its workspace's own egress. + # The verifier runs on the substrate's network with nothing between + # it and the hosts it dials — and an allowlist declared for the + # environment is one the verifier inherits, so it is refused too. + effective = verifier.network_mode or environment.network_mode + if effective == "allowlist": + reasons.append( + "verifier network_mode='allowlist' (only the agent's is enforceable)" + ) if environment.os not in (None, "linux"): reasons.append(f"environment.os={environment.os!r}") if environment.tpu: diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index 953ec9d95..f731a0650 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -275,6 +275,7 @@ def test_declared_workspace_policy_is_translated(tmp_path) -> None: "env": {"TOKEN": "abc"}, "agent_env": {}, "workdir": "/srv/app", + "allowed_hosts": None, "user": None, # Per phase, because Harbor's are: a task may restrict the agent and # still verify as root. @@ -297,11 +298,24 @@ def test_no_network_is_honored_and_allowlist_refused(tmp_path) -> None: encoding="utf-8", ) - # no-network is deliverable (a sandboxed workspace); allowlist is not. + # no-network is deliverable (a sandboxed workspace). An allowlist is too, + # but only for the agent: its workspace has an egress to apply it to. The + # verifier runs on the substrate's own network, and an allowlist declared + # for the environment is one the verifier inherits. assert harbor_load.unsupported_features(isolated) == [] assert harbor_load.workspace_policy(isolated)["network"] is False + assert harbor_load.workspace_policy(isolated)["allowed_hosts"] == [] assert "allowlist" in " ".join(harbor_load.unsupported_features(filtered)) + agent_only = _write_harbor_task(tmp_path, "agent-only") + (agent_only / "task.toml").write_text( + 'schema_version = "1.3"\n\n[task]\nname = "demo/agent-only"\n\n' + '[agent]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', + encoding="utf-8", + ) + assert harbor_load.unsupported_features(agent_only) == [] + assert harbor_load.workspace_policy(agent_only)["allowed_hosts"] == ["pypi.org"] + def test_tasks_with_different_policies_get_separate_envs(tmp_path) -> None: # Same build context, different declared workdir: one env serves one From f076c1850faa5bed80b0cf7ba9286fac1c938556 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:45:30 -0700 Subject: [PATCH 16/33] feat(integrations): take every task's network through the workspace's own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Public was the one policy that still shared the substrate's network, which meant "the agent may reach any host" also meant "the agent may reach the channel grading it, the ssh daemon serving it, and anything else the substrate is running". Every task in the sample is public, so the exposure was the common case rather than the exception. Public now means every host reached the same way as any other policy: through the workspace's own way out. The agent's network ends up with exactly one listener, the proxy. Bounding it moves the verifier too. Harbor grades a service by talking to it — telnet to the VM the agent booted, git clone from the server it configured — and those listen in the workspace now, so the verifier joins its user and network namespaces while keeping this mount namespace: /tests and the verdict stay where the workspace cannot reach them, and the ids map through, so it can still chown what it untars. Behind that boundary it needs the way out too, since Harbor's verifiers commonly install their own tooling first. The bridge is waited for rather than merely spawned. Sessions start as soon as the sandbox does, and a session that opens with a package install against a port not yet bound reads a connection refused as a network that does not work — which looked like a per-host failure until the timing was the thing being measured. Verified on every shape: public, allowlisted, no-network, non-root image, a task whose verifier reaches a service the agent started, and configure-git-webserver end to end. --- hud/environment/egress.py | 24 ++++++++++++---- hud/environment/workspace.py | 12 +++++++- integrations/harbor/_adapt.py | 32 ++++++++++++++++++++-- integrations/harbor/_load.py | 7 ++++- integrations/harbor/tests/test_contract.py | 4 ++- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/hud/environment/egress.py b/hud/environment/egress.py index ba0aedeed..821bca54c 100644 --- a/hud/environment/egress.py +++ b/hud/environment/egress.py @@ -20,6 +20,7 @@ from __future__ import annotations +import asyncio import contextlib import http.client import logging @@ -75,6 +76,7 @@ async def bridged(reader, writer): async def main(): server = await asyncio.start_server(bridged, "127.0.0.1", int(sys.argv[2])) + print("ready", flush=True) async with server: await server.serve_forever() @@ -200,7 +202,7 @@ def __init__(self, socket_path: Path | str, allowed: Collection[str]) -> None: self.allowed = frozenset(allowed) self._server: _UnixProxyServer | None = None self._thread: threading.Thread | None = None - self._bridge: subprocess.Popen[bytes] | None = None + self._bridge: asyncio.subprocess.Process | None = None def start(self) -> None: """Serve the policy on the unix socket. Idempotent.""" @@ -215,15 +217,19 @@ def start(self) -> None: self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() - def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: + async def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: """Offer the proxy on the loopback of *pid*'s network namespace. The bridge joins that namespace and nothing else, so it keeps this filesystem — which is how it reaches a socket the workspace cannot. + + Returns once it is accepting rather than once it is spawned: a session + starting in between finds the port refused, which a task opening with + a package install reads as a network that does not work. """ nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" - self._bridge = subprocess.Popen( - [ + self._bridge = await asyncio.create_subprocess_exec( + *[ nsenter, "--target", str(pid), @@ -237,9 +243,14 @@ def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: self.socket_path, str(port), ], - stdout=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, ) + assert self._bridge.stdout is not None + try: + await asyncio.wait_for(self._bridge.stdout.readline(), 30.0) + except TimeoutError: + LOGGER.warning("the workspace's way out did not come up in time") def environment(self, port: int = BRIDGE_PORT) -> dict[str, str]: """Proxy variables for a session, in the spellings clients read.""" @@ -256,7 +267,8 @@ def environment(self, port: int = BRIDGE_PORT) -> dict[str, str]: def stop(self) -> None: """Take the route away.""" if self._bridge is not None: - self._bridge.kill() + with contextlib.suppress(ProcessLookupError): + self._bridge.kill() self._bridge = None if self._server is not None: self._server.shutdown() diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 03656957e..43bf5363d 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -424,6 +424,16 @@ def __init__( # and two that each started a sandbox would not share one. self._sandbox_lock = asyncio.Lock() + def egress_environment(self) -> dict[str, str]: + """Proxy variables for a process joining this workspace's network. + + Anything entering it is behind the same boundary as a session and has + the same one way out — including the verifier, which reaches a service + the agent started by joining, and would otherwise find a network that + refuses everything it tries to install. + """ + return self._egress.environment() if self._egress is not None else {} + @property def owns_netns(self) -> bool: """Whether the workspace has a network of its own. @@ -851,7 +861,7 @@ async def _start_sandbox(self) -> int: if self.allowed_hosts: self._egress = Egress(self._credentials_dir() / "egress.sock", self.allowed_hosts) self._egress.start() - self._egress.attach(pid) + await self._egress.attach(pid) return pid async def _sandbox_error(self) -> str: diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index f10ae3616..c507ff82b 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -575,7 +575,7 @@ async def _run_harbor_task() -> AsyncGenerator[Any, Any]: _hide_grading_dirs() try: answer = yield (task_dir / "instruction.md").read_text(encoding="utf-8") - yield await _grade(task_dir, workdir, answer) + yield await _grade(task_dir, workdir, answer, workspace) finally: _hide_grading_dirs() # Harbor's agent phase is one continuous session, so a service the @@ -630,7 +630,9 @@ def _reset_dir(path: Path) -> None: shutil.rmtree(child) -async def _grade(task_dir: Path, workdir: Path, answer: Any) -> dict[str, Any]: +async def _grade( + task_dir: Path, workdir: Path, answer: Any, workspace: Workspace | None = None +) -> dict[str, Any]: logs = LOGS # The agent shares the container, so restore the verifier from the baked # (masked) copy before running it. @@ -666,6 +668,32 @@ async def _grade(task_dir: Path, workdir: Path, answer: Any) -> dict[str, Any]: "--clear-groups", *argv, ] + joins_workspace_net = ( + workspace is not None and workspace.owns_netns and config.network("verifier") + ) + if joins_workspace_net and (sandbox := await workspace.sandbox_pid()) is not None: # type: ignore[union-attr] + # Harbor grades a service by talking to it — telnet to the VM the agent + # booted, git clone from the server it configured. Those listen in the + # workspace's network now, so the verifier is run there: its user and + # network namespaces, and nothing else. Keeping this mount namespace is + # the point — /tests and the verdict are here, where the workspace + # cannot reach them — and the ids map through, so it stays able to + # chown what it untars. + # Behind the workspace's boundary now, so it takes the workspace's way + # out: Harbor's verifiers commonly install their own tooling first. + verifier_env.update(workspace.egress_environment()) # type: ignore[union-attr] + nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" + argv = [ + nsenter, + "--target", + str(sandbox), + "--user", + "--net", + "--preserve-credentials", + "--", + *argv, + ] + severed = not config.network("verifier") if severed and (bwrap := usable_bwrap()) is None: raise RuntimeError( diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py index 5e1f9e5e9..4229c341f 100644 --- a/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -14,6 +14,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError +from hud.environment.egress import ANY_HOST from hud.eval import Task, Taskset from hud.eval.runtime import RuntimeConfig, RuntimeGPU, RuntimeResources from hud.utils.naming import normalize_environment_name @@ -139,7 +140,11 @@ def allowed_hosts(self, role: str) -> frozenset[str] | None: if mode == "no-network": return frozenset() if mode != "allowlist": - return None + # Public still means every host, but reached the same way as any + # other policy: through the workspace's own way out. Sharing the + # substrate's network would make "public" mean the substrate's + # services too — the channel that grades the rollout among them. + return frozenset({ANY_HOST}) declared = [*self.environment.allowed_hosts, *self.phase(role).allowed_hosts] return frozenset(declared) diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index f731a0650..d790c8f34 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -275,7 +275,9 @@ def test_declared_workspace_policy_is_translated(tmp_path) -> None: "env": {"TOKEN": "abc"}, "agent_env": {}, "workdir": "/srv/app", - "allowed_hosts": None, + # Public, but reached through the workspace's own way out rather than + # by sharing the substrate's network. + "allowed_hosts": ["*"], "user": None, # Per phase, because Harbor's are: a task may restrict the agent and # still verify as root. From c6b16b23ac01a98acc29c47b76c9e67e338b4d87 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:04:58 -0700 Subject: [PATCH 17/33] fix(environment): land a session in the sandbox's own working directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nsenter opens a directory given to --wd before it joins any namespace, so the path is resolved on the substrate rather than inside the sandbox. The default guest path exists only in the sandbox's mount namespace, so every session on a workspace that took bwrap's defaults died at startup with "nsenter: cannot open /workspace" — the shell never ran, and the failure looked like the command's. Bare --wd is the sandbox's own working directory, which bwrap set to the guest path when it built it, so a session lands where it always meant to. That leaves nothing for enter_argv's cwd to select: it named a directory nsenter could only misread, and no caller passed one. Harbor was unaffected — its guest path is the image's WORKDIR, which the substrate has too — which is why the sample kept scoring. --- hud/environment/tests/test_workspace.py | 7 ++++++- hud/environment/workspace.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index f6908e760..b6a62b2a9 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -305,7 +305,12 @@ def test_sessions_join_one_sandbox_rather_than_each_making_its_own( # The user namespace has to be joined first: it is what confers the # privilege to join the others in a container given no extra capability. assert first.index("--user") < min(first.index("--mount"), first.index("--pid")) - assert "--wd=/app" in first + # The sandbox's own working directory, never a path spelled out here: + # nsenter opens a directory it is given *before* it joins anything, out + # where a guest path the substrate does not have does not resolve at all. + assert "--wd" in first + assert not any(argument.startswith("--wd=") for argument in first) + assert "/app" not in first[: first.index("--")] @pytest.mark.asyncio diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 43bf5363d..9af0cf0a7 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -727,7 +727,6 @@ def enter_argv( pid: int, command: str | None = None, *, - cwd: str | None = None, env: Mapping[str, str] | None = None, tty: bool = False, ) -> list[str]: @@ -743,6 +742,11 @@ def enter_argv( sharing sandbox is already in this process's netns, and that netns belongs to an outer user namespace: once joined to bwrap's, we hold no authority there and rejoining fails outright. + + The sandbox's own working directory is the only one a session can be + started in, so there is no ``cwd`` to choose here: a directory named + to ``nsenter`` is opened *before* it joins anything, out where a guest + path that exists only inside the sandbox does not resolve. """ nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" argv = [ @@ -758,7 +762,7 @@ def enter_argv( # otherwise a session would run on the substrate's, which is the # network the workspace was given a policy to keep it off. *(("--net",) if self.owns_netns else ()), - f"--wd={cwd if cwd is not None else self._guest_path}", + "--wd", "--", ] # Unlike the bwrap path, the drop goes *inside*: joining namespaces From 7b2d4e5d467fb01a5ee3d8a31c0fd0cf53ae847a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:05:17 -0700 Subject: [PATCH 18/33] feat(environment): let a bounded workspace reach the services it declares MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workspace with a network of its own cannot address the substrate at all. That is the point where the control channel is concerned, but it applies just as much to services the environment itself runs: a database the task depends on, an API it is meant to call, anything an image starts. Until now those were as unreachable as the channel, so a bounded workspace could only be given to tasks that needed nothing but the internet. A Peer is one of those services, handed back deliberately and at the address the task expects rather than wherever it happens to listen. The proxy's bridge already offered one socket on the workspace's loopback; it now offers one per route, and a peer is a socket that splices to a substrate address instead of applying a host policy. Names come from a hosts file the sandbox mounts over its own, so a task that says "db:5432" finds it there; 127.0.0.1 is used wherever the port is free, since a task that says "localhost:8080" means that one, and only a second service on the same port moves. Peers are reached directly rather than through the proxy — sent there, they would be resolved out on the substrate, where the name means nothing and the address is something else — so no_proxy names them. A workspace that may reach no host is told of no proxy at all, rather than being pointed at one that was never started. --- hud/environment/__init__.py | 2 + hud/environment/egress.py | 301 ++++++++++++++++++------ hud/environment/tests/test_workspace.py | 80 +++++++ hud/environment/workspace.py | 41 +++- 4 files changed, 351 insertions(+), 73 deletions(-) diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index 002eef168..356d26b62 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -22,6 +22,7 @@ from hud.capabilities import Capability from hud.utils.modules import iter_modules +from .egress import Peer from .env import Answer, Environment from .integration import Integration from .workspace import DEFAULT_SYSTEM_MOUNTS, Mount, MountKind, Workspace @@ -95,6 +96,7 @@ def load_environment( "Integration", "Mount", "MountKind", + "Peer", "Workspace", "load_environment", ] diff --git a/hud/environment/egress.py b/hud/environment/egress.py index 821bca54c..f10a5d493 100644 --- a/hud/environment/egress.py +++ b/hud/environment/egress.py @@ -1,16 +1,18 @@ -"""The one way out of a bounded workspace, and the policy on it. +"""The ways out of a bounded workspace, and the policy on them. A workspace with its own network namespace has no route anywhere — not to the internet, and not to whatever else the substrate is running, including the -control channel that grades it. Egress is given back deliberately, through a -proxy that sees every connection and applies the task's declared policy. - -The proxy listens on a unix socket, so reaching it is a question of the -filesystem rather than the network: a bridge runs in the workspace's *network* -namespace while keeping the substrate's *mount* namespace, so it can see the -socket the workspace itself cannot, and offers it as an ordinary proxy port on -the workspace's loopback. Nothing is bound into the workspace, and nothing in -it can address the substrate. +control channel that grades it. Two kinds of route are given back +deliberately: hosts on the internet, through a proxy that sees every +connection and applies the task's declared policy, and :class:`Peer` services +the environment itself runs, each forwarded to the address the task expects. + +Both listen on unix sockets, so reaching them is a question of the filesystem +rather than the network: a bridge runs in the workspace's *network* namespace +while keeping the substrate's *mount* namespace, so it can see sockets the +workspace itself cannot, and offers them as ordinary ports on the workspace's +loopback. Nothing is bound into the workspace, and nothing in it can address +the substrate except through one of these. Request parsing is the standard library's. A hand-rolled request-line parser gets keep-alive, chunked bodies and header framing wrong in ways that surface @@ -23,6 +25,7 @@ import asyncio import contextlib import http.client +import json import logging import os import select @@ -33,12 +36,13 @@ import sys import threading import urllib.parse +from dataclasses import dataclass from http.server import BaseHTTPRequestHandler +from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Collection - from pathlib import Path + from collections.abc import Collection, Sequence LOGGER = logging.getLogger("hud.environment.egress") @@ -54,8 +58,12 @@ #: an egress proxy is ordinary infrastructure, unlike a control channel. BRIDGE_PORT = 3128 +#: Run inside the workspace's network namespace, one listener per route out. +#: Its argument is ``[[host, port, socket], ...]``; every listener is bound +#: before it says it is ready, since a session that starts in between finds +#: the port refused. _BRIDGE = """ -import asyncio, sys +import asyncio, json, sys async def splice(reader, writer): try: @@ -70,20 +78,88 @@ async def splice(reader, writer): except Exception: pass -async def bridged(reader, writer): - up_reader, up_writer = await asyncio.open_unix_connection(sys.argv[1]) - await asyncio.gather(splice(reader, up_writer), splice(up_reader, writer)) +def bridged(path): + async def handle(reader, writer): + try: + up_reader, up_writer = await asyncio.open_unix_connection(path) + except OSError: + writer.close() + return + await asyncio.gather(splice(reader, up_writer), splice(up_reader, writer)) + return handle async def main(): - server = await asyncio.start_server(bridged, "127.0.0.1", int(sys.argv[2])) + servers = [ + await asyncio.start_server(bridged(path), host, port) + for host, port, path in json.loads(sys.argv[1]) + ] print("ready", flush=True) - async with server: - await server.serve_forever() + await asyncio.gather(*(server.serve_forever() for server in servers)) asyncio.run(main()) """ +@dataclass(frozen=True, slots=True) +class Peer: + """A substrate service a bounded workspace is allowed to reach. + + A workspace with its own network cannot address the substrate at all — + that is what makes it bounded — so a service the environment runs is as + unreachable from it as the control channel. A peer hands one of them back, + at the address the task expects rather than wherever it happens to listen: + ``name`` and ``port`` are what the workspace calls it, ``target`` where it + actually answers outside (its own port on the substrate's loopback, unless + something else is said). + """ + + name: str + port: int + target: tuple[str, int] | None = None + + @property + def address(self) -> tuple[str, int]: + """Where the service actually listens, on the substrate.""" + return self.target or ("127.0.0.1", self.port) + + +def bind_addresses(peers: Sequence[Peer]) -> dict[str, str]: + """Which loopback address each peer answers on inside the workspace. + + ``127.0.0.1`` wherever the port is free, because a task that says + ``localhost:6379`` means that one. Two peers cannot both hold a port + there, so the second moves down 127.0.0.0/8 and is reached by its name — + which is how a task naming several services addresses them anyway. + """ + taken: set[tuple[str, int]] = set() + addresses: dict[str, str] = {} + for peer in peers: + if peer.name in addresses: + raise ValueError(f"two peers are called {peer.name!r}") + for index in range(1, 256): + host = f"127.0.0.{index}" + if (host, peer.port) not in taken: + break + else: + raise ValueError(f"too many peers on port {peer.port}") + taken.add((host, peer.port)) + addresses[peer.name] = host + return addresses + + +def hosts_text(peers: Sequence[Peer], base: str) -> str: + """*base* — the substrate's ``/etc/hosts`` — plus a line per peer. + + Names resolve for what runs in the workspace's *mount* namespace, which + is its sessions. Anything joining only the network namespace (the Harbor + verifier does, to reach a service the agent started) still reaches a peer + at its address, but not by its name. + """ + addresses = bind_addresses(peers) + lines = "".join(f"{addresses[peer.name]}\t{peer.name}\n" for peer in peers) + return f"{base.rstrip(chr(10))}\n{lines}" if base.strip() else lines + + def permitted(host: str | None, allowed: Collection[str]) -> bool: """Whether *host* is in *allowed*, by exact match or as a subdomain.""" if not host: @@ -93,6 +169,23 @@ def permitted(host: str | None, allowed: Collection[str]) -> bool: return any(host == entry or host.endswith(f".{entry}") for entry in allowed) +def _relay(one: socket.socket, other: socket.socket, timeout: float = 300.0) -> None: + """Copy bytes between two connected sockets until either end is done.""" + while True: + ready, _, _ = select.select([one, other], [], [], timeout) + if not ready: + return + for source in ready: + target = other if source is one else one + try: + data = source.recv(65536) + if not data: + return + target.sendall(data) + except OSError: + return + + class _Proxy(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" allowed: Collection[str] = () @@ -120,21 +213,8 @@ def do_CONNECT(self) -> None: return self.send_response(200, "Connection established") self.end_headers() - client = self.connection with upstream: - while True: - ready, _, _ = select.select([client, upstream], [], [], 300) - if not ready: - return - for source in ready: - target = upstream if source is client else client - try: - data = source.recv(65536) - if not data: - return - target.sendall(data) - except OSError: - return + _relay(self.connection, upstream) def _forward(self) -> None: parts = urllib.parse.urlsplit(self.path) @@ -180,7 +260,21 @@ def _forward(self) -> None: do_OPTIONS = _forward -class _UnixProxyServer(socketserver.ThreadingUnixStreamServer): +class _Forward(socketserver.BaseRequestHandler): + """One peer's socket: everything on it goes to that service, unread.""" + + target: tuple[str, int] = ("127.0.0.1", 0) + + def handle(self) -> None: + try: + upstream = socket.create_connection(self.target, timeout=15) + except OSError: + return + with upstream: + _relay(self.request, upstream) + + +class _UnixServer(socketserver.ThreadingUnixStreamServer): daemon_threads = True def get_request(self) -> tuple[socket.socket, tuple[str, int]]: @@ -190,43 +284,88 @@ def get_request(self) -> tuple[socket.socket, tuple[str, int]]: class Egress: - """A workspace's route out, and the policy applied to it. + """A workspace's routes out, and the policy applied to them. + + ``allowed`` is the set of internet hosts a session may reach — + ``{ANY_HOST}`` for all of them, and an empty set for a workspace that may + reach none. ``peers`` are substrate services it may reach whatever the + host policy says: they are named by the task rather than dialed by the + agent, so reaching one is not a question the allowlist answers. - ``allowed`` is the set of hosts a session may reach — ``{ANY_HOST}`` for - all of them. An empty set is a workspace that can reach nothing, which is - also what not starting one at all means. + Every socket lives in ``socket_dir``, which must be somewhere the + workspace cannot see: a socket it could connect to directly would be a + route out that skips all of this. """ - def __init__(self, socket_path: Path | str, allowed: Collection[str]) -> None: - self.socket_path = str(socket_path) + def __init__( + self, + socket_dir: Path | str, + allowed: Collection[str], + peers: Sequence[Peer] = (), + ) -> None: + self.socket_dir = Path(socket_dir) self.allowed = frozenset(allowed) - self._server: _UnixProxyServer | None = None - self._thread: threading.Thread | None = None + self.peers = tuple(peers) + self._servers: list[tuple[_UnixServer, Path]] = [] self._bridge: asyncio.subprocess.Process | None = None + @property + def socket_path(self) -> Path: + """The proxy's socket — the way out to the hosts policy allows.""" + return self.socket_dir / "egress.sock" + + def _peer_socket(self, index: int) -> Path: + # By position rather than by name: a peer's name comes from the task, + # and a task does not get to choose paths in here. + return self.socket_dir / f"peer-{index}.sock" + def start(self) -> None: - """Serve the policy on the unix socket. Idempotent.""" - if self._server is not None: + """Serve the policy, and each declared peer, on a socket. Idempotent.""" + if self._servers: return + self.socket_dir.mkdir(parents=True, exist_ok=True) + if self.allowed: + self._serve( + self.socket_path, type("_ScopedProxy", (_Proxy,), {"allowed": self.allowed}) + ) + for index, peer in enumerate(self.peers): + self._serve( + self._peer_socket(index), + type("_PeerForward", (_Forward,), {"target": peer.address}), + ) + + def _serve(self, path: Path, handler: type[socketserver.BaseRequestHandler]) -> None: with contextlib.suppress(FileNotFoundError): - os.unlink(self.socket_path) - os.makedirs(os.path.dirname(self.socket_path) or ".", exist_ok=True) - handler = type("_ScopedProxy", (_Proxy,), {"allowed": self.allowed}) - self._server = _UnixProxyServer(self.socket_path, handler) - os.chmod(self.socket_path, 0o600) - self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) - self._thread.start() + os.unlink(path) + server = _UnixServer(str(path), handler) + os.chmod(path, 0o600) + threading.Thread(target=server.serve_forever, daemon=True).start() + self._servers.append((server, path)) + + def _bridge_spec(self, port: int) -> list[tuple[str, int, str]]: + """Where each route out is offered inside the workspace.""" + addresses = bind_addresses(self.peers) + return [ + *([("127.0.0.1", port, str(self.socket_path))] if self.allowed else []), + *( + (addresses[peer.name], peer.port, str(self._peer_socket(index))) + for index, peer in enumerate(self.peers) + ), + ] async def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: - """Offer the proxy on the loopback of *pid*'s network namespace. + """Offer every route on the loopback of *pid*'s network namespace. The bridge joins that namespace and nothing else, so it keeps this - filesystem — which is how it reaches a socket the workspace cannot. + filesystem — which is how it reaches sockets the workspace cannot. Returns once it is accepting rather than once it is spawned: a session starting in between finds the port refused, which a task opening with a package install reads as a network that does not work. """ + spec = self._bridge_spec(port) + if not spec: + return nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" self._bridge = await asyncio.create_subprocess_exec( *[ @@ -240,8 +379,7 @@ async def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: sys.executable, "-c", _BRIDGE, - self.socket_path, - str(port), + json.dumps(spec), ], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, @@ -250,33 +388,56 @@ async def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: try: await asyncio.wait_for(self._bridge.stdout.readline(), 30.0) except TimeoutError: - LOGGER.warning("the workspace's way out did not come up in time") + LOGGER.warning("the workspace's ways out did not come up in time") def environment(self, port: int = BRIDGE_PORT) -> dict[str, str]: - """Proxy variables for a session, in the spellings clients read.""" + """Proxy variables for a session, in the spellings clients read. + + Empty where no host is permitted: pointing a client at a proxy that + is not there turns "this task has no network" into a connection error + on the first hop, which reads as a broken one instead. + """ + if not self.allowed: + return {} url = f"http://127.0.0.1:{port}" + # Peers are reached directly, on the loopback the bridge binds them + # to; sent through the proxy instead, they would be resolved out here, + # where the name means nothing and the address is something else. + # Listed one by one rather than as 127.0.0.0/8, which most clients + # (curl among them) match literally instead of as a network. + addresses = bind_addresses(self.peers) + bypass = ",".join( + dict.fromkeys(["127.0.0.1", "localhost", *addresses, *addresses.values()]) + ) return { "http_proxy": url, "https_proxy": url, "HTTP_PROXY": url, "HTTPS_PROXY": url, - "no_proxy": "127.0.0.1,localhost", - "NO_PROXY": "127.0.0.1,localhost", + "no_proxy": bypass, + "NO_PROXY": bypass, } def stop(self) -> None: - """Take the route away.""" + """Take the routes away.""" if self._bridge is not None: with contextlib.suppress(ProcessLookupError): self._bridge.kill() self._bridge = None - if self._server is not None: - self._server.shutdown() - self._server.server_close() - self._server = None - self._thread = None - with contextlib.suppress(FileNotFoundError): - os.unlink(self.socket_path) - - -__all__ = ["ANY_HOST", "BRIDGE_PORT", "Egress", "permitted"] + for server, path in self._servers: + server.shutdown() + server.server_close() + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + self._servers = [] + + +__all__ = [ + "ANY_HOST", + "BRIDGE_PORT", + "Egress", + "Peer", + "bind_addresses", + "hosts_text", + "permitted", +] diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index b6a62b2a9..8504a871f 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -389,6 +389,86 @@ def test_making_a_network_and_joining_it_are_the_same_question( assert ("--net" in ws.enter_argv(7, "true")) is owns +def test_a_peer_answers_at_the_address_the_task_expects() -> None: + """A task that names a service says where it expects to find it. Placed + anywhere else, the task's own client configuration points at nothing.""" + from hud.environment.egress import Peer, bind_addresses + + # One peer per port is the ordinary case: it is at localhost, which is + # what a task saying "localhost:5432" or "http://localhost:8080" means. + single = bind_addresses([Peer("db", 5432), Peer("api", 8080)]) + assert single == {"db": "127.0.0.1", "api": "127.0.0.1"} + + # Two services cannot both hold one port there, so the second moves — and + # is still reached by its name, which is how a task addresses two anyway. + both = bind_addresses([Peer("primary", 5432), Peer("replica", 5432)]) + assert both == {"primary": "127.0.0.1", "replica": "127.0.0.2"} + + with pytest.raises(ValueError, match="two peers are called"): + bind_addresses([Peer("db", 5432), Peer("db", 6379)]) + + +def test_a_peers_name_is_added_to_the_substrates_hosts_rather_than_replacing_it() -> None: + """Dropping the substrate's entries would cost the workspace localhost.""" + from hud.environment.egress import Peer, hosts_text + + text = hosts_text([Peer("db", 5432)], "127.0.0.1\tlocalhost\n::1\tip6-localhost\n") + + assert "127.0.0.1\tlocalhost" in text + assert "::1\tip6-localhost" in text + assert text.endswith("127.0.0.1\tdb\n") + + +@pytest.mark.asyncio +async def test_a_declared_peer_is_a_name_sessions_resolve( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A peer reached only by port is not at the address the task expects, so + the workspace carries its own hosts file — and only when it has a network + of its own, since otherwise the service is already at its real address.""" + from hud.environment.egress import Peer + + ws = Workspace(tmp_path / "root", peers=[Peer("db", 5432)], allowed_hosts={"pypi.org"}) + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + ws._prepare_runtime() + + argv = ws.bwrap_argv(["true"]) + hosts = Path(argv[argv.index("/etc/hosts") - 1]) + assert argv[argv.index("/etc/hosts") - 2] == "--ro-bind" + assert "127.0.0.1\tdb\n" in hosts.read_text() + # Bound over /etc/hosts, so every session reads it whatever its identity. + assert hosts.stat().st_mode & 0o044 + + sharing = Workspace(tmp_path / "shared", peers=[Peer("db", 5432)], network=True) + monkeypatch.setattr(sharing, "_bwrap", "/usr/bin/bwrap") + sharing._prepare_runtime() + assert "/etc/hosts" not in sharing.bwrap_argv(["true"]) + + await ws.stop() + await sharing.stop() + + +def test_a_peer_is_reached_directly_rather_than_through_the_proxy() -> None: + """The proxy resolves names out on the substrate, where a peer's name + means nothing and its address is something else entirely.""" + from hud.environment.egress import Egress, Peer + + egress = Egress("/tmp/unused", {"pypi.org"}, [Peer("db", 5432), Peer("replica", 5432)]) + bypass = egress.environment()["no_proxy"].split(",") + + assert "db" in bypass and "replica" in bypass + assert "127.0.0.1" in bypass and "127.0.0.2" in bypass + + +def test_a_workspace_that_reaches_no_host_is_told_of_no_proxy() -> None: + """Pointing a client at a proxy that was never started turns "this task + has no network" into a connection failure on the first hop.""" + from hud.environment.egress import Egress, Peer + + assert Egress("/tmp/unused", set(), [Peer("db", 5432)]).environment() == {} + assert Egress("/tmp/unused", {"pypi.org"}).environment()["https_proxy"].endswith(":3128") + + def test_a_host_is_permitted_by_name_or_as_a_subdomain() -> None: from hud.environment.egress import ANY_HOST, permitted diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 9af0cf0a7..ac72f98fd 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -19,7 +19,7 @@ import asyncssh -from hud.environment.egress import Egress +from hud.environment.egress import Egress, Peer, hosts_text from hud.utils.process import create_process_group_exec if sys.platform != "win32": # the pty a session runs on has no Windows analogue @@ -327,6 +327,7 @@ def __init__( mounts: Sequence[Mount] = (), network: bool = False, allowed_hosts: Collection[str] | None = None, + peers: Sequence[Peer] = (), env: Mapping[str, str] | None = None, system_mounts: Sequence[Mount] | None = None, guest_path: str = "/workspace", @@ -363,7 +364,17 @@ def __init__( #: addressable from inside, and no session can reach a host the task #: did not declare. self.allowed_hosts = None if allowed_hosts is None else frozenset(allowed_hosts) + #: Substrate services the workspace may reach, each at the address the + #: task expects. A workspace with a network of its own cannot address + #: the substrate at all, so anything the environment itself runs — a + #: database the task depends on, an API it is meant to call — has to be + #: named here to exist for it. Nothing to do where sessions share the + #: substrate's network: the services are already at those addresses. + self.peers: tuple[Peer, ...] = tuple(peers) self._egress: Egress | None = None + # The workspace's own /etc/hosts (the substrate's, plus its peers), + # materialized alongside the session keys when there is one to write. + self._hosts_path: Path | None = None self.env: dict[str, str] = dict(env or {}) self._system_mounts: tuple[Mount, ...] = tuple( system_mounts if system_mounts is not None else DEFAULT_SYSTEM_MOUNTS, @@ -507,6 +518,8 @@ def _prepare_runtime(self) -> None: os.lchown(self.root, self._shell_uid, self._shell_uid) self._host_key, self._host_pubkey_str = self._load_or_generate_host_key() self._authorized_keys_path = self._ensure_authorized_keys_file() + if self.peers and self.owns_netns and self._bwrap is not None: + self._hosts_path = self._write_hosts() self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._sock.bind((self._ssh_host, self._ssh_port)) @@ -717,6 +730,11 @@ def bwrap_argv( argv.extend(["--bind", str(self.root), self._guest_path]) for m in self.mounts: argv.extend(m.to_bwrap_args()) + if self._hosts_path is not None: + # Last, so it survives whatever the caller mounted over /etc: a + # peer the task can address by port but not by name is not at the + # address the task expects. + argv.extend(Mount("ro", src=str(self._hosts_path), dst="/etc/hosts").to_bwrap_args()) argv.extend(["--chdir", target_cwd]) argv.append("--") argv.extend(_payload_argv(command, full_env, ctty=tty)) @@ -862,8 +880,8 @@ async def _start_sandbox(self) -> int: await self.discard_sandbox() raise RuntimeError(f"the sandbox never became ready: {reason}") self._sandbox_init = pid - if self.allowed_hosts: - self._egress = Egress(self._credentials_dir() / "egress.sock", self.allowed_hosts) + if self.owns_netns and (self.allowed_hosts or self.peers): + self._egress = Egress(self._credentials_dir(), self.allowed_hosts or (), self.peers) self._egress.start() await self._egress.attach(pid) return pid @@ -964,6 +982,22 @@ def _credentials_dir(self) -> Path: self._cred_dir = Path(tempfile.mkdtemp(prefix="ssh-")) return self._cred_dir + def _write_hosts(self) -> Path: + """The workspace's own ``/etc/hosts``: the substrate's, plus its peers. + + World-readable, unlike the rest of the credentials directory: it is + bound over ``/etc/hosts`` inside the sandbox, where every session — + including one dropped to an id of its own — resolves names from it. + """ + substrate = Path("/etc/hosts") + path = self._credentials_dir() / "hosts" + path.write_text( + hosts_text(self.peers, substrate.read_text() if substrate.is_file() else ""), + encoding="utf-8", + ) + path.chmod(0o644) + return path + def _load_or_generate_host_key(self) -> tuple[asyncssh.SSHKey, str]: if self._ssh_host_key_path is not None: key = asyncssh.read_private_key(self._ssh_host_key_path) @@ -1263,5 +1297,6 @@ async def relay_output( "DEFAULT_SYSTEM_MOUNTS", "Mount", "MountKind", + "Peer", "Workspace", ] From 93659e214879ef9c0c2bd84d4c90a27fe430260c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:43:12 -0700 Subject: [PATCH 19/33] fix(environment): stop the proxy forwarding framing it has already undone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transfer-Encoding is hop-by-hop, and the proxy was passing it through. The response body comes back from http.client already de-chunked, so the client was handed plain bytes under a header telling it to look for chunk lengths: curl reports "illegal or missing hexadecimal sequence in chunked-encoding" and the body is whatever survives that. Chunked is what a server reaches for when it cannot know the length up front — a generated index, a streamed archive — so this was worst for exactly the kind of fetch a task does before it can start work. --- hud/environment/egress.py | 13 +++++- hud/environment/tests/test_workspace.py | 54 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/hud/environment/egress.py b/hud/environment/egress.py index f10a5d493..80b51f49c 100644 --- a/hud/environment/egress.py +++ b/hud/environment/egress.py @@ -50,8 +50,19 @@ ANY_HOST = "*" #: Headers that belong to one hop and must not be forwarded to the next. +#: ``transfer-encoding`` among them: the response body is read back already +#: de-chunked, so passing the upstream's framing along leaves the client +#: looking for chunk headers in what is now plain bytes. _HOP_BY_HOP = frozenset( - {"connection", "proxy-connection", "keep-alive", "te", "trailers", "upgrade"} + { + "connection", + "proxy-connection", + "keep-alive", + "te", + "trailers", + "transfer-encoding", + "upgrade", + } ) #: The proxy port offered on the workspace's loopback. 3128 is unremarkable — diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 8504a871f..ef7cf0e36 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -5,6 +5,7 @@ import asyncio import itertools import os +import shutil import sys import tempfile import time @@ -460,6 +461,59 @@ def test_a_peer_is_reached_directly_rather_than_through_the_proxy() -> None: assert "127.0.0.1" in bypass and "127.0.0.2" in bypass +def test_the_proxy_does_not_forward_framing_it_has_already_undone() -> None: + """The body comes back de-chunked, so passing the upstream's chunked + framing along with it leaves the client reading chunk headers out of plain + bytes — an index that fails halfway rather than an obvious error.""" + import socket as socket_mod + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from hud.environment.egress import Egress + + class Chunked(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(b"5\r\nhello\r\n0\r\n\r\n") + + def log_message(self, *_: object) -> None: + pass + + upstream = HTTPServer(("127.0.0.1", 0), Chunked) + threading.Thread(target=upstream.serve_forever, daemon=True).start() + port = upstream.server_address[1] + # Not the pytest tmp dir: a unix socket path is capped near 104 bytes. + sockets = Path(tempfile.mkdtemp(dir="/tmp")) + egress = Egress(sockets, {"127.0.0.1"}) + egress.start() + try: + client = socket_mod.socket(socket_mod.AF_UNIX, socket_mod.SOCK_STREAM) + client.settimeout(10) + client.connect(str(egress.socket_path)) + client.sendall(f"GET http://127.0.0.1:{port}/ HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n".encode()) + received = b"" + while b"hello" not in received: + chunk = client.recv(4096) + if not chunk: + break + received += chunk + client.close() + finally: + egress.stop() + upstream.shutdown() + upstream.server_close() + shutil.rmtree(sockets, ignore_errors=True) + + headers, _, body = received.partition(b"\r\n\r\n") + assert b"200" in headers + assert b"transfer-encoding" not in headers.lower() + assert body == b"hello" + + def test_a_workspace_that_reaches_no_host_is_told_of_no_proxy() -> None: """Pointing a client at a proxy that was never started turns "this task has no network" into a connection failure on the first hop.""" From 6205b0823cf85f8df62a2b23d99d95a4a8b75ea8 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:43:23 -0700 Subject: [PATCH 20/33] fix(integrations): hold the verifier to the hosts it declared, not the agent's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verifier joins the workspace's network namespace so it can reach a service the agent started — telnet to the VM it booted, git clone from the server it configured. It was then taking the workspace's way out too, which applies the *agent's* allowlist. A task that restricted its agent to one host therefore restricted its grader to the same one, and Harbor's verifiers almost all begin by installing their own tooling: Err:1 http://ports.ubuntu.com/ubuntu-ports noble InRelease 403 Forbidden [IP: 127.0.0.1 3128] /tests/test.sh: line 23: uv: command not found The verifier then scored the task 0.0 for a reason its author never wrote, and nothing in the run said so. A visitor — a process in the workspace's network that is not one of its sessions — now gets a way out of its own, on its own port, under the policy its own phase declared. It exists only while the visitor runs: the agent's sessions share that network, so a second and more permissive route standing open would be one the agent could simply take instead of the one it was given. During the agent phase the port refuses connections. That makes a verifier-declared allowlist enforceable rather than ignored, so the refusal that existed because it was not is gone, and each phase's hosts are carried separately in the workspace policy that keys environments. --- hud/environment/egress.py | 55 +++++++++++++-------- hud/environment/workspace.py | 40 +++++++++++---- integrations/harbor/_adapt.py | 57 ++++++++++++---------- integrations/harbor/_load.py | 28 +++-------- integrations/harbor/tests/test_contract.py | 38 +++++++++------ 5 files changed, 124 insertions(+), 94 deletions(-) diff --git a/hud/environment/egress.py b/hud/environment/egress.py index 80b51f49c..20c2494e5 100644 --- a/hud/environment/egress.py +++ b/hud/environment/egress.py @@ -69,6 +69,13 @@ #: an egress proxy is ordinary infrastructure, unlike a control channel. BRIDGE_PORT = 3128 +#: Where a visitor's way out is offered instead. A visitor joins the +#: workspace's network without being one of its sessions, and is held to its +#: own policy rather than the sessions' — so this is a second proxy, on a +#: second port, and it exists only while the visitor is there. Standing open +#: it would be a route the agent could take in place of the one it was given. +VISITOR_PORT = 3129 + #: Run inside the workspace's network namespace, one listener per route out. #: Its argument is ``[[host, port, socket], ...]``; every listener is bound #: before it says it is ready, since a session that starts in between finds @@ -171,6 +178,29 @@ def hosts_text(peers: Sequence[Peer], base: str) -> str: return f"{base.rstrip(chr(10))}\n{lines}" if base.strip() else lines +def proxy_environment(port: int, peers: Sequence[Peer] = ()) -> dict[str, str]: + """Proxy variables for a process on a workspace's loopback. + + In the spellings clients read, and with the peers left out of them: a peer + is reached directly, on the loopback the bridge binds it to, because sent + through the proxy it would be resolved on the substrate, where the name + means nothing and the address is something else. Listed one by one rather + than as 127.0.0.0/8, which most clients (curl among them) match literally + instead of as a network. + """ + url = f"http://127.0.0.1:{port}" + addresses = bind_addresses(peers) + bypass = ",".join(dict.fromkeys(["127.0.0.1", "localhost", *addresses, *addresses.values()])) + return { + "http_proxy": url, + "https_proxy": url, + "HTTP_PROXY": url, + "HTTPS_PROXY": url, + "no_proxy": bypass, + "NO_PROXY": bypass, + } + + def permitted(host: str | None, allowed: Collection[str]) -> bool: """Whether *host* is in *allowed*, by exact match or as a subdomain.""" if not host: @@ -402,32 +432,13 @@ async def attach(self, pid: int, port: int = BRIDGE_PORT) -> None: LOGGER.warning("the workspace's ways out did not come up in time") def environment(self, port: int = BRIDGE_PORT) -> dict[str, str]: - """Proxy variables for a session, in the spellings clients read. + """Proxy variables for what this serves. Empty where no host is permitted: pointing a client at a proxy that is not there turns "this task has no network" into a connection error on the first hop, which reads as a broken one instead. """ - if not self.allowed: - return {} - url = f"http://127.0.0.1:{port}" - # Peers are reached directly, on the loopback the bridge binds them - # to; sent through the proxy instead, they would be resolved out here, - # where the name means nothing and the address is something else. - # Listed one by one rather than as 127.0.0.0/8, which most clients - # (curl among them) match literally instead of as a network. - addresses = bind_addresses(self.peers) - bypass = ",".join( - dict.fromkeys(["127.0.0.1", "localhost", *addresses, *addresses.values()]) - ) - return { - "http_proxy": url, - "https_proxy": url, - "HTTP_PROXY": url, - "HTTPS_PROXY": url, - "no_proxy": bypass, - "NO_PROXY": bypass, - } + return proxy_environment(port, self.peers) if self.allowed else {} def stop(self) -> None: """Take the routes away.""" @@ -446,9 +457,11 @@ def stop(self) -> None: __all__ = [ "ANY_HOST", "BRIDGE_PORT", + "VISITOR_PORT", "Egress", "Peer", "bind_addresses", "hosts_text", "permitted", + "proxy_environment", ] diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index ac72f98fd..50b56db8a 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -19,7 +19,7 @@ import asyncssh -from hud.environment.egress import Egress, Peer, hosts_text +from hud.environment.egress import VISITOR_PORT, Egress, Peer, hosts_text, proxy_environment from hud.utils.process import create_process_group_exec if sys.platform != "win32": # the pty a session runs on has no Windows analogue @@ -28,7 +28,7 @@ import termios if TYPE_CHECKING: - from collections.abc import Collection, Mapping, Sequence + from collections.abc import AsyncIterator, Collection, Mapping, Sequence from hud.capabilities import Capability @@ -435,15 +435,35 @@ def __init__( # and two that each started a sandbox would not share one. self._sandbox_lock = asyncio.Lock() - def egress_environment(self) -> dict[str, str]: - """Proxy variables for a process joining this workspace's network. - - Anything entering it is behind the same boundary as a session and has - the same one way out — including the verifier, which reaches a service - the agent started by joining, and would otherwise find a network that - refuses everything it tries to install. + @contextlib.asynccontextmanager + async def visiting(self, allowed: Collection[str]) -> AsyncIterator[dict[str, str]]: + """A way out for a process joining this network without being a session. + + Yields the proxy variables it should run under. A visitor is behind the + same boundary as a session — it is in the same namespace — but it is + not the party the sessions' policy was written about: a grader reaching + a service the agent started answers to what *it* was allowed, not to + what the agent was. Sharing the sessions' way out instead holds it to + the agent's allowlist, which for a grader that installs its own tooling + first means it fails before it asserts anything. + + Open only for as long as the visitor runs, and on a port of its own. + The agent's sessions share this network, so a second and more permissive + way out that stood open would be one the agent could simply take. """ - return self._egress.environment() if self._egress is not None else {} + pid = await self.sandbox_pid() + if pid is None or not self.owns_netns or not allowed: + yield {} + return + egress = Egress(self._credentials_dir() / "visit", allowed) + egress.start() + try: + await egress.attach(pid, VISITOR_PORT) + # The peers are the workspace's, bound by its own bridge: a visitor + # reaches them at those addresses, so they stay out of its proxy. + yield proxy_environment(VISITOR_PORT, self.peers) + finally: + egress.stop() @property def owns_netns(self) -> bool: diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index c507ff82b..0ec3512ed 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -668,32 +668,6 @@ async def _grade( "--clear-groups", *argv, ] - joins_workspace_net = ( - workspace is not None and workspace.owns_netns and config.network("verifier") - ) - if joins_workspace_net and (sandbox := await workspace.sandbox_pid()) is not None: # type: ignore[union-attr] - # Harbor grades a service by talking to it — telnet to the VM the agent - # booted, git clone from the server it configured. Those listen in the - # workspace's network now, so the verifier is run there: its user and - # network namespaces, and nothing else. Keeping this mount namespace is - # the point — /tests and the verdict are here, where the workspace - # cannot reach them — and the ids map through, so it stays able to - # chown what it untars. - # Behind the workspace's boundary now, so it takes the workspace's way - # out: Harbor's verifiers commonly install their own tooling first. - verifier_env.update(workspace.egress_environment()) # type: ignore[union-attr] - nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" - argv = [ - nsenter, - "--target", - str(sandbox), - "--user", - "--net", - "--preserve-credentials", - "--", - *argv, - ] - severed = not config.network("verifier") if severed and (bwrap := usable_bwrap()) is None: raise RuntimeError( @@ -762,7 +736,36 @@ async def run_tests() -> ProcessGroup: if stray != -1: os.close(stray) - return await _grade_with_verifier(config, logs, answer, run_tests) + if workspace is None or severed or not workspace.owns_netns: + return await _grade_with_verifier(config, logs, answer, run_tests) + sandbox = await workspace.sandbox_pid() + if sandbox is None: + return await _grade_with_verifier(config, logs, answer, run_tests) + + # Harbor grades a service by talking to it — telnet to the VM the agent + # booted, git clone from the server it configured. Those listen in the + # workspace's network now, so the verifier is run there: its user and + # network namespaces, and nothing else. Keeping this mount namespace is the + # point — /tests and the verdict are here, where the workspace cannot reach + # them — and the ids map through, so it stays able to chown what it untars. + async with workspace.visiting(config.allowed_hosts("verifier")) as visitor_env: + # Behind the workspace's boundary now, so it needs a way out — one of + # its own, since Harbor's verifiers commonly install their own tooling + # first and the hosts they may reach are the ones the *verifier* + # declared, not the ones the agent was held to. + verifier_env.update(visitor_env) + nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" + argv = [ + nsenter, + "--target", + str(sandbox), + "--user", + "--net", + "--preserve-credentials", + "--", + *argv, + ] + return await _grade_with_verifier(config, logs, answer, run_tests) # ─── verifier grading and docker plumbing ─────────────────────────────── diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py index 4229c341f..b940d1288 100644 --- a/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -128,13 +128,15 @@ def network(self, role: str) -> bool: """ return "no-network" not in (self.environment.network_mode, self.phase(role).network_mode) - def allowed_hosts(self, role: str) -> frozenset[str] | None: - """Which hosts *role* may reach, or None to leave its network alone. + def allowed_hosts(self, role: str) -> frozenset[str]: + """Which hosts *role* may reach. A phase declaring ``allowlist`` may reach the hosts it names and nothing else — including nothing else on the substrate, which is where the harness serving it lives. ``public`` names no hosts and so permits all of them; ``no-network`` is the empty set, which permits none. + Every phase gets an answer: a task that says nothing about its network + is still held to something, rather than sharing the substrate's. """ mode = self.phase(role).network_mode or self.environment.network_mode or "public" if mode == "no-network": @@ -234,9 +236,10 @@ def workspace_policy(task_dir: Path) -> dict[str, Any]: # Sorted rather than a set: the policy is hashed to key environments, # so it has to serialize, and two tasks naming the same hosts in a # different order declare the same thing. - "allowed_hosts": ( - None if (hosts := config.allowed_hosts("agent")) is None else sorted(hosts) - ), + "allowed_hosts": sorted(config.allowed_hosts("agent")), + # The verifier's are its own. Grouped on as well, since a task whose + # grader may reach different hosts is a different environment. + "verifier_allowed_hosts": sorted(config.allowed_hosts("verifier")), "agent_user": config.phase_user("agent"), "verifier_user": config.phase_user("verifier"), } @@ -286,21 +289,6 @@ def unsupported_features(task_dir: Path) -> list[str]: environment, agent, verifier = config.environment, config.agent, config.verifier reasons: list[str] = [] - for role, mode in ( - ("environment", environment.network_mode), - ("agent", agent.network_mode), - ("verifier", verifier.network_mode), - ): - if mode == "allowlist" and role in ("environment", "verifier"): - # The agent's allowlist is applied by its workspace's own egress. - # The verifier runs on the substrate's network with nothing between - # it and the hosts it dials — and an allowlist declared for the - # environment is one the verifier inherits, so it is refused too. - effective = verifier.network_mode or environment.network_mode - if effective == "allowlist": - reasons.append( - "verifier network_mode='allowlist' (only the agent's is enforceable)" - ) if environment.os not in (None, "linux"): reasons.append(f"environment.os={environment.os!r}") if environment.tpu: diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index d790c8f34..a1d174404 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -278,6 +278,8 @@ def test_declared_workspace_policy_is_translated(tmp_path) -> None: # Public, but reached through the workspace's own way out rather than # by sharing the substrate's network. "allowed_hosts": ["*"], + # And the verifier's are its own, not a copy of the agent's. + "verifier_allowed_hosts": ["*"], "user": None, # Per phase, because Harbor's are: a task may restrict the agent and # still verify as root. @@ -286,29 +288,33 @@ def test_declared_workspace_policy_is_translated(tmp_path) -> None: } -def test_no_network_is_honored_and_allowlist_refused(tmp_path) -> None: +def test_each_phase_is_held_to_the_hosts_it_declared(tmp_path) -> None: + """An allowlist written for one phase must not become the other's. The + agent's applied to the verifier is a grader that cannot install its own + tooling — a zero the task's author never wrote.""" isolated = _write_harbor_task(tmp_path, "isolated") (isolated / "task.toml").write_text( 'schema_version = "1.3"\n\n[task]\nname = "demo/isolated"\n\n' '[environment]\nnetwork_mode = "no-network"\n', encoding="utf-8", ) - filtered = _write_harbor_task(tmp_path, "filtered") - (filtered / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/filtered"\n\n' - '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', - encoding="utf-8", - ) - - # no-network is deliverable (a sandboxed workspace). An allowlist is too, - # but only for the agent: its workspace has an egress to apply it to. The - # verifier runs on the substrate's own network, and an allowlist declared - # for the environment is one the verifier inherits. assert harbor_load.unsupported_features(isolated) == [] assert harbor_load.workspace_policy(isolated)["network"] is False assert harbor_load.workspace_policy(isolated)["allowed_hosts"] == [] - assert "allowlist" in " ".join(harbor_load.unsupported_features(filtered)) + assert harbor_load.workspace_policy(isolated)["verifier_allowed_hosts"] == [] + # Declared for the whole environment: both phases named those hosts. + shared = _write_harbor_task(tmp_path, "shared") + (shared / "task.toml").write_text( + 'schema_version = "1.3"\n\n[task]\nname = "demo/shared"\n\n' + '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', + encoding="utf-8", + ) + assert harbor_load.unsupported_features(shared) == [] + assert harbor_load.workspace_policy(shared)["allowed_hosts"] == ["pypi.org"] + assert harbor_load.workspace_policy(shared)["verifier_allowed_hosts"] == ["pypi.org"] + + # Declared for the agent alone: the verifier said nothing, so it is public. agent_only = _write_harbor_task(tmp_path, "agent-only") (agent_only / "task.toml").write_text( 'schema_version = "1.3"\n\n[task]\nname = "demo/agent-only"\n\n' @@ -317,6 +323,7 @@ def test_no_network_is_honored_and_allowlist_refused(tmp_path) -> None: ) assert harbor_load.unsupported_features(agent_only) == [] assert harbor_load.workspace_policy(agent_only)["allowed_hosts"] == ["pypi.org"] + assert harbor_load.workspace_policy(agent_only)["verifier_allowed_hosts"] == ["*"] def test_tasks_with_different_policies_get_separate_envs(tmp_path) -> None: @@ -627,7 +634,6 @@ def test_final_stage_reads_only_what_the_shipped_image_declares() -> None: @pytest.mark.parametrize( ("declaration", "expected"), [ - ('[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', "allowlist"), ('[environment.healthcheck]\ncommand = "curl -sf localhost/health"\n', "healthcheck"), ( '[[environment.mcp_servers]]\nname = "db"\nurl = "http://localhost:9000/sse"\n', @@ -658,11 +664,11 @@ async def test_a_refused_task_never_reaches_a_build_context(tmp_path) -> None: task = _write_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text( 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', + '[environment.healthcheck]\ncommand = "curl -sf localhost/health"\n', encoding="utf-8", ) - with pytest.raises(NotImplementedError, match="allowlist"): + with pytest.raises(NotImplementedError, match="healthcheck"): await harbor.adapt(tmp_path, build=False) From 00ce09188cfedab98edde427237c2085577b947e Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 01:28:22 -0700 Subject: [PATCH 21/33] refactor(integrations): make Harbor use the HUD environment boundary --- docs/v6/advanced/harbor-convert.mdx | 23 +- hud/environment/tests/test_workspace.py | 30 + hud/environment/workspace.py | 99 ++- integrations/__init__.py | 4 +- integrations/harbor/__init__.py | 31 +- integrations/harbor/_adapt.py | 776 ++------------------- integrations/harbor/_export.py | 19 +- integrations/harbor/_load.py | 357 +++++----- integrations/harbor/_runtime.py | 388 +++++++++++ integrations/harbor/tests/test_contract.py | 166 +++-- integrations/harbor/tests/test_harbor.py | 1 + 11 files changed, 821 insertions(+), 1073 deletions(-) create mode 100644 integrations/harbor/_runtime.py diff --git a/docs/v6/advanced/harbor-convert.mdx b/docs/v6/advanced/harbor-convert.mdx index 4750d12d7..6a9df8aae 100644 --- a/docs/v6/advanced/harbor-convert.mdx +++ b/docs/v6/advanced/harbor-convert.mdx @@ -30,6 +30,7 @@ directly - one row per task dir (`id` = the dir name), sharing one declarative ```python from integrations import harbor +from hud.eval import DockerRuntime assert harbor.detect("./terminal-bench") taskset = harbor.load("./terminal-bench") @@ -41,28 +42,24 @@ for task in taskset: Each row carries what the task declared: `[metadata]` as `columns` (difficulty, category, tags - the platform's filter/leaderboard facets) and `[environment]` cpu/memory/gpu as `runtime_config`. Time budgets stay off the -row, since they bound the *rollout* rather than the substrate - read one with -`harbor.agent_timeout(task_dir)` and pass it as `rollout_timeout`. +row, since they bound the *rollout* rather than the substrate; pass the desired +budget as `rollout_timeout` when running. ## Run Harbor tasks Build the images once, then place the rows anywhere: ```python -images = await harbor.adapt("./terminal-bench") # {env name: image ref} -taskset = harbor.load("./terminal-bench", images=images) # rows carry the image -job = await taskset.run(agent, runtime=harbor.docker_runtime()) +taskset = await harbor.adapt("./terminal-bench") +job = await taskset.run(agent, runtime=DockerRuntime()) ``` -`harbor.docker_runtime()` is a `DockerRuntime` that permits nested -namespaces: an adapted image sandboxes *inside itself* to keep the baked -tests and the verdict away from the agent, which plain images neither need -nor should be given. +`DockerRuntime` supplies the inner workspace sandbox needed by adapted images; +the setting is automatic for all HUD Docker environments. -The mapping is a value you hold - nothing is written into the dataset - so -there is no cache to go stale. `adapt(path, push="registry.io/acme")` pushes -instead; `hud deploy` of the generated contexts under `.hud-adapt/` works the -same way, since both run the image's own CMD. +`adapt(path, push="registry.io/acme")` pushes instead and returns rows bound to +the pushed images. It also writes the generated contexts under `.hud-adapt/`, +which `hud deploy` can use directly. The image also carries what the task declared about its environment: `[environment.env]`, `workdir` and `user` become `ENV`, `WORKDIR` and `USER` diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index ef7cf0e36..d603715e5 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -390,6 +390,36 @@ def test_making_a_network_and_joining_it_are_the_same_question( assert ("--net" in ws.enter_argv(7, "true")) is owns +def test_process_builders_can_apply_a_phase_policy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ws = Workspace(tmp_path / "root", network=True, env={"AGENT_ONLY": "yes"}) + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + + verifier = ws.bwrap_argv( + ["true"], + env={"VERIFIER_ONLY": "yes"}, + inherit_workspace_env=False, + network=False, + mount_hosts=False, + ) + entered = ws.enter_argv( + 7, + ["true"], + env={"VERIFIER_ONLY": "yes"}, + identity=None, + inherit_workspace_env=False, + preserve_credentials=True, + ) + + assert "--unshare-net" in verifier + assert "AGENT_ONLY=yes" not in verifier + assert "VERIFIER_ONLY=yes" in verifier + assert "--preserve-credentials" in entered + assert "AGENT_ONLY=yes" not in entered + assert "VERIFIER_ONLY=yes" in entered + + def test_a_peer_answers_at_the_address_the_task_expects() -> None: """A task that names a service says where it expects to find it. Placed anywhere else, the task's own client configuration points at nothing.""" diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 50b56db8a..2176781de 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -40,9 +40,8 @@ # Set once the first Workspace logs the missing-bwrap notice (avoid per-instance spam). #: bwrap's usability is a property of the host/container, so probe once per -#: process: an installed bwrap that cannot create namespaces (a container -#: whose seccomp profile blocks unprivileged userns) would otherwise fail -#: every session instead of falling back. +#: process: an installed bwrap that cannot create namespaces would otherwise +#: fail every session instead of falling back. _bwrap_usable: bool | None = None @@ -84,8 +83,8 @@ def usable_bwrap() -> str | None: if not _bwrap_usable: LOGGER.warning( "bwrap is installed but cannot create namespaces (%s); sessions will " - "run WITHOUT isolation. Allow unprivileged user namespaces (e.g. " - "docker --security-opt seccomp=unconfined) to enable it.", + "run WITHOUT isolation. The container runtime must allow " + "unprivileged user namespaces to enable it.", probe.stderr.decode("utf-8", "replace").strip()[:120], ) except (OSError, subprocess.SubprocessError): @@ -404,9 +403,9 @@ def __init__( if require_isolation and self._bwrap is None: raise RuntimeError( "isolation was required but bwrap cannot sandbox here: install " - "bubblewrap and allow unprivileged user namespaces (e.g. run the " - "container with --security-opt seccomp=unconfined). Refusing to " - "serve sessions that would silently run unisolated." + "bubblewrap and use a container runtime that allows unprivileged " + "user namespaces. Refusing to serve sessions that would silently " + "run unisolated." ) self._ssh_host_key_path = host_key_path self._ssh_authorized_client_keys = list(authorized_client_keys or []) @@ -710,8 +709,12 @@ def bwrap_argv( cwd: str | None = None, env: Mapping[str, str] | None = None, inherit_host_env: bool = True, + inherit_workspace_env: bool = True, info_fd: int | None = None, userns_block_fd: int | None = None, + network: bool | None = None, + mount_hosts: bool = True, + isolate_processes: bool = True, tty: bool = False, ) -> list[str]: """Argv that runs ``command`` inside bwrap. Raises if bwrap unavailable. @@ -727,19 +730,26 @@ def bwrap_argv( raise RuntimeError("bwrap not available on this host") target_cwd = cwd if cwd is not None else self._guest_path base_env = _without_harness_config(os.environ) if inherit_host_env else {} - full_env = {**base_env, **self.env, **(env or {})} + workspace_env = self.env if inherit_workspace_env else {} + full_env = {**base_env, **workspace_env, **(env or {})} + owns_netns = self.owns_netns if network is None else not network argv: list[str] = [ self._bwrap, "--die-with-parent", # Blocking means this side installs the map, so the namespace has # to be ours to map: --unshare-user, not the best-effort form. "--unshare-user" if userns_block_fd is not None else "--unshare-user-try", - "--unshare-pid", - "--unshare-ipc", - "--unshare-uts", - "--unshare-cgroup-try", ] - if self.owns_netns: + if isolate_processes: + argv.extend( + [ + "--unshare-pid", + "--unshare-ipc", + "--unshare-uts", + "--unshare-cgroup-try", + ] + ) + if owns_netns: argv.append("--unshare-net") if info_fd is not None: argv.extend(["--info-fd", str(info_fd)]) @@ -750,7 +760,7 @@ def bwrap_argv( argv.extend(["--bind", str(self.root), self._guest_path]) for m in self.mounts: argv.extend(m.to_bwrap_args()) - if self._hosts_path is not None: + if mount_hosts and self._hosts_path is not None: # Last, so it survives whatever the caller mounted over /etc: a # peer the task can address by port but not by name is not at the # address the task expects. @@ -766,6 +776,10 @@ def enter_argv( command: str | None = None, *, env: Mapping[str, str] | None = None, + identity: int | None | Literal["workspace"] = "workspace", + inherit_workspace_env: bool = True, + preserve_credentials: bool = False, + no_new_privs: bool = True, tty: bool = False, ) -> list[str]: """Argv that runs ``command`` inside the sandbox *pid* belongs to. @@ -801,36 +815,65 @@ def enter_argv( # network the workspace was given a policy to keep it off. *(("--net",) if self.owns_netns else ()), "--wd", + *(("--preserve-credentials",) if preserve_credentials else ()), "--", ] # Unlike the bwrap path, the drop goes *inside*: joining namespaces # needs the privileges the dropped uid does not have. - argv.extend(self._drop_argv()) - argv.extend(_payload_argv(command, self._full_env(env), ctty=tty)) + if identity == "workspace": + argv.extend(self._drop_argv(no_new_privs=no_new_privs)) + elif identity is not None: + argv.extend(self._drop_argv(identity, no_new_privs=no_new_privs)) + argv.extend( + _payload_argv( + command, + self._full_env(env, include_workspace_env=inherit_workspace_env), + ctty=tty, + ) + ) return argv - def _full_env(self, env: Mapping[str, str] | None = None) -> dict[str, str]: + def _full_env( + self, + env: Mapping[str, str] | None = None, + *, + include_workspace_env: bool = True, + ) -> dict[str, str]: """The environment a session starts from. Dropped sessions get the minimal one built for the wall; otherwise the serving process's environment carries through, less HUD's own. """ proxy = self._egress.environment() if self._egress is not None else {} - if self._drops_privileges(): + if include_workspace_env and self._drops_privileges(): return {**(self._session_env() or {}), **proxy, **(env or {})} - return {**_without_harness_config(os.environ), **proxy, **self.env, **(env or {})} + workspace_env = self.env if include_workspace_env else {} + return {**_without_harness_config(os.environ), **proxy, **workspace_env, **(env or {})} - def _drop_argv(self) -> list[str]: + def _drop_argv(self, uid: int | None = None, *, no_new_privs: bool = True) -> list[str]: """The ``setpriv`` prefix that drops to ``shell_uid``, if it applies.""" - if not self._drops_privileges(): + if uid is None: + if not self._drops_privileges(): + return [] + uid = self._shell_uid + elif not (_is_root() and sys.platform == "linux" and self._setpriv() is not None): return [] setpriv = self._setpriv() - assert setpriv is not None # guaranteed by _drops_privileges - uid = str(self._shell_uid) - # --no-new-privs: without it a setuid binary (or passwordless sudo) - # inside the workspace would let the dropped shell regain root and - # read the secrets the wall protects. - return [setpriv, "--reuid", uid, "--regid", uid, "--clear-groups", "--no-new-privs", "--"] + assert setpriv is not None + uid_text = str(uid) + # Without this, a setuid binary (or passwordless sudo) inside the + # workspace could let the dropped shell regain root. + no_new_privs_argv = ["--no-new-privs"] if no_new_privs else [] + return [ + setpriv, + "--reuid", + uid_text, + "--regid", + uid_text, + "--clear-groups", + *no_new_privs_argv, + "--", + ] # ─── the sandbox sessions share ─────────────────────────────────────── diff --git a/integrations/__init__.py b/integrations/__init__.py index 4085fdc87..6cbbfd176 100644 --- a/integrations/__init__.py +++ b/integrations/__init__.py @@ -6,6 +6,6 @@ Each format implements that contract (``load`` / ``environment``) and keeps an ergonomic function surface (``harbor.load(...)``) plus format extras: ``harbor.detect`` recognizes the layout, ``harbor.adapt`` packages the -constructor into container images, ``harbor.export`` is the reverse -direction. +constructor into container images and returns bound rows, and +``harbor.export`` is the reverse direction. """ diff --git a/integrations/harbor/__init__.py b/integrations/harbor/__init__.py index ae2468844..629c4f03b 100644 --- a/integrations/harbor/__init__.py +++ b/integrations/harbor/__init__.py @@ -11,15 +11,14 @@ Harbor's agent works *inside* its container, so :func:`environment` (the :class:`~hud.environment.Integration` constructor) is meaningful only in -there — :func:`adapt` packages it: one HUD-speaking image per env group -whose CMD serves ``harbor.environment``, and the same rows then run on any -container placement:: +there. :func:`adapt` builds the images and returns the loaded rows with those +images bound to them:: - await harbor.adapt("./tasks") # local images - job = await harbor.load("./tasks").run(agent, runtime=DockerRuntime()) + taskset = await harbor.adapt("./tasks") + job = await taskset.run(agent, runtime=DockerRuntime()) - await harbor.adapt("./tasks", push="registry.io/x") # or hud deploy the - job = await harbor.load("./tasks").run(agent, runtime=HUDRuntime()) # contexts + taskset = await harbor.adapt("./tasks", push="registry.io/x") + job = await taskset.run(agent, runtime=HUDRuntime()) Plus :func:`export`, the reverse direction (HUD tasks -> Harbor folders). Compose-based and prebuilt-``docker_image`` tasks are not supported yet. @@ -37,12 +36,13 @@ from hud.environment import Environment from hud.eval import Taskset -from ._adapt import adapt, docker_runtime, environment -from ._export import ALLOWED_PROTOCOLS, CONTROL_PORT, DEFAULT_ANSWER_FILE, export -from ._load import agent_timeout, detect, grouped, load +from ._adapt import adapt +from ._export import export +from ._load import detect, load +from ._runtime import environment -class Harbor(Integration): +class _Harbor(Integration): """The :class:`~hud.environment.Integration` contract for Harbor.""" name = "harbor" @@ -54,20 +54,13 @@ def environment(self, ref: str | Path, *, name: str | None = None) -> Environmen return environment(ref, name=name) -integration = Harbor() +integration = _Harbor() __all__ = [ - "ALLOWED_PROTOCOLS", - "CONTROL_PORT", - "DEFAULT_ANSWER_FILE", - "Harbor", "adapt", - "agent_timeout", "detect", - "docker_runtime", "environment", "export", - "grouped", "integration", "load", ] diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py index 0ec3512ed..c4b77d1c3 100644 --- a/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -1,90 +1,28 @@ -"""Adapted Harbor images: build them, and serve from inside them. - -A Harbor task's environment *is* a container, so its environment constructor -is only meaningful in there. :func:`adapt` packages it: one image per env -group whose CMD serves the HUD control channel from inside, which is the one -assumption every container placement makes — so the same rows run on local -docker, a cloud sandbox, or a platform deploy. - -:func:`environment` is what those images serve, by module reference:: - - CMD[ - "hud", - "serve", - "harbor:environment", - "--arg", - "ref=/media/hud/tasks", - "--arg", - "name=", - ] - -Everything it needs is container-local: the workspace is the image's working -directory, and grading runs each task's ``tests/test.sh`` in place. The -sandbox exists to keep the graded material — the baked tasks under the -harness's own tree, and the verifier's verdict — outside the agent's -namespace, and to sever the network when a task declares no-network:: - - images = await harbor.adapt("./harbor_tasks", push="registry.io/acme") - taskset = harbor.load("./harbor_tasks", images=images) - await taskset.run(agent, runtime=DaytonaRuntime()) -""" +"""Build adapted Harbor images.""" from __future__ import annotations -import asyncio -import contextlib import json import logging -import math -import os -import pwd import re import shlex import shutil -from collections.abc import ( # noqa: TC003 - env.template resolves at runtime - AsyncGenerator, - Awaitable, - Callable, -) from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING -from hud.environment import Environment, Mount -from hud.environment.workspace import Workspace, install_identity_map, usable_bwrap -from hud.eval import DockerRuntime from hud.utils.docker import docker as _docker -from hud.utils.process import ProcessGroup, create_process_group_exec -from ._load import ( - DEFAULT_VERIFIER_TIMEOUT, - HUD_ROOT, - TaskConfig, - final_stage, - grouped, - hash_directory, - slugify, - unsupported_features, - workspace_policy, -) +from ._load import HUD_ROOT, HarborTask, _load, _task_groups, hash_directory -LOGGER = logging.getLogger(__name__) +if TYPE_CHECKING: + from hud.eval import Taskset -#: Harbor's absolute conventions. ``VERIFIER_LOGS`` holds the verdict, so it -#: is masked from agent sessions (see :func:`environment`) and is the only -#: place :func:`read_reward` trusts. -LOGS = Path("/logs") -VERIFIER_LOGS = LOGS / "verifier" -TESTS = Path("/tests") +LOGGER = logging.getLogger(__name__) -#: Task config is untrusted: these bound what may reach a generated directive. _DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*") - -#: The interpreter the serving venv is built on. The layer copies this -#: integration into that venv's site-packages by path, so the two must agree. SERVING_PYTHON = "3.12" -#: Build-context entries never copied into adapted contexts. _CONTEXT_IGNORE_NAMES = ( "__pycache__", "*.pyc", @@ -98,39 +36,21 @@ _INSTALL_SH = """\ #!/bin/sh -# Install a self-contained hud venv under __HUD_ROOT__: bootstrap uv (which brings its -# own managed Python), never touching the image's Python or site-packages. -# Needs network and one of: uv / curl / wget / pip (+ apt-get/apk for bare -# images with no downloader). set -eu -# Everything HUD installs goes under __HUD_ROOT__ — the uv binary, its managed -# interpreter, that interpreter's shims — because that tree is the one the -# workspace keeps out of the graded party's namespace. At uv's defaults this lands -# in the invoking user's home, which is *inside* the task's filesystem: the -# agent then finds a HUD runtime the task never declared, and shims pointing -# into the mask that resolve to nothing. Containing it is what makes the single -# single hiding of that tree sufficient, rather than state to clean up after. -# The cache is the same argument, minus the keeping: nothing reads it after this -# script, so it is never written. -export UV_INSTALL_DIR=__HUD_ROOT__/bin \\ - UV_PYTHON_INSTALL_DIR=__HUD_ROOT__/python \\ - UV_PYTHON_BIN_DIR=__HUD_ROOT__/bin \\ - UV_NO_CACHE=1 \\ +export UV_INSTALL_DIR=__HUD_ROOT__/bin \ + UV_PYTHON_INSTALL_DIR=__HUD_ROOT__/python \ + UV_PYTHON_BIN_DIR=__HUD_ROOT__/bin \ + UV_NO_CACHE=1 \ XDG_CONFIG_HOME=__HUD_ROOT__/config export PATH="__HUD_ROOT__/bin:$PATH" command -v uv >/dev/null 2>&1 || { - { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \\ - || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \\ + { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \ + || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \ || apk add --no-cache curl ca-certificates - { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \\ - || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \\ + { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \ + || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \ || pip install -q -U uv } -# bubblewrap backs the workspace sandbox, which keeps the baked tasks and the -# verifier's verdict outside the agent's namespace (and severs the network for -# a no-network task). The container must also permit nested namespaces, which -# HUD's placements grant (seccomp/systempaths unconfined); where they do not, -# the env refuses to serve rather than serving forgeable rollouts. command -v bwrap >/dev/null 2>&1 \ || { apt-get update -qq && apt-get install -y -qq bubblewrap; } \ || apk add --no-cache bubblewrap \ @@ -139,23 +59,12 @@ uv venv __HUD_ROOT__/venv --python __PYTHON__ uv pip install --python __HUD_ROOT__/venv/bin/python __HUD_REQUIREMENT__ -# ─── the agent's toolchain ─── -# Harbor installs the agent *into* the container, and that step provisions what -# the agent needs (BaseInstalledAgent.SYSTEM_PACKAGES: python3, pip, git, curl) -# before the agent phase. HUD's agent is external and installs nothing, so -# without this an adapted image hands the agent a barer machine than Harbor -# would for the same task — a difference in the harness, scored as if it were a -# difference in the model. -# -# Build time, not per rollout: baked into the content-addressed layer once and -# reused by every rollout. Only what the image actually lacks is installed, so -# an image shipping its own Python keeps exactly that Python. apt_pkgs="" apk_pkgs="" -for spec in \\ - "python3|python3 python3-venv|python3" \\ - "pip3|python3-pip|py3-pip" \\ - "git|git|git" \\ +for spec in \ + "python3|python3 python3-venv|python3" \ + "pip3|python3-pip|py3-pip" \ + "git|git|git" \ "curl|curl ca-certificates|curl ca-certificates" do if command -v "${spec%%|*}" >/dev/null 2>&1; then continue; fi @@ -164,29 +73,19 @@ apk_pkgs="$apk_pkgs ${rest##*|}" done if [ -n "$apt_pkgs" ]; then - { apt-get update -qq && apt-get install -y -qq $apt_pkgs; } \\ - || apk add --no-cache $apk_pkgs \\ + { apt-get update -qq && apt-get install -y -qq $apt_pkgs; } \ + || apk add --no-cache $apk_pkgs \ || echo "warning: could not provision the agent toolchain:$apt_pkgs" fi """ _LAYER = """ -# ─── HUD adaptation layer: serve the control channel from inside ─── -# (generated by harbor.adapt) -# Installed as root: the base stage may end on a non-root USER that could not -# write __HUD_ROOT__. The task's own declared user, if any, is restored below. +# HUD adaptation layer USER root COPY _hud __HUD_ROOT__ RUN sh __HUD_ROOT__/install.sh -# This integration ships outside the hud distribution, so it comes from the -# build context rather than the index — which also means the code serving the -# image is the revision that adapted it, not whatever the index resolves to. COPY _hud_harbor __HUD_ROOT__/venv/lib/python__PYTHON__/site-packages/harbor -# The CLI's update check has no user to prompt here, and its cache lives in the -# invoking user's home — inside the task's filesystem. A container is fresh -# every rollout, so the cache is never warm: left on, it calls PyPI on the -# rollout's critical path and leaves HUD state in the graded filesystem. ENV HUD_SKIP_VERSION_CHECK=1 EXPOSE 8765 __DECLARED__ENTRYPOINT [] @@ -197,101 +96,27 @@ def _validated_user(declared: str | int | None, source_user: str | None) -> str | None: - """A phase's identity: what it declared, else the image's own ``USER``.""" + """Validate a phase identity before it reaches a generated directive.""" user = str(declared) if declared is not None else source_user if user is None: return None - # Untrusted input: the task config and the image's own Dockerfile. if not all(_DOCKER_USER.fullmatch(part) for part in user.split(":", 1)): raise ValueError(f"declared user {user!r} is not a usable user[:group]") return user -def phase_uid(task_dir: Path, role: str) -> int | None: - """The id *role* runs as in this image, or None to run as the harness does. - - Harbor runs the agent and the verifier as the identity the task declares - for each, and its own harness is not subject to either. Serving from - inside the image, a ``USER`` directive would demote the harness too — - leaving it unable to create ``/tests`` at the filesystem root, or its own - state under ``/hud``. So the identity is applied where the phase runs. - - A name the image does not have is an error rather than a fall back to - root: running a phase with more privilege than the task granted is the - thing this exists to prevent. - """ - user = _validated_user(workspace_policy(task_dir)[f"{role}_user"], _image_user(task_dir)) - if user is None: - return None - name = user.split(":", 1)[0] - if name.isdigit(): - return int(name) - try: - return pwd.getpwnam(name).pw_uid - except KeyError as error: - raise ValueError(f"declared user {user!r} does not exist in this image") from error - - -def phase_home(uid: int | None) -> str | None: - """*uid*'s home, for a phase that is about to drop to it. - - Dropping ids without one leaves the phase pointed at the harness's home, - which it cannot write — and a verifier that installs its own tooling - (Harbor's commonly do) fails on that alone. - """ - if uid is None: - return None - with contextlib.suppress(KeyError): - return pwd.getpwuid(uid).pw_dir - return None - - -def _image_user(task_dir: Path) -> str | None: - """The ``USER`` the adapted image recorded for itself, if any. - - Read from the layer rather than the Dockerfile: by serve time the build - context is gone, and the image's own declaration still has to be honoured - for a task whose ``task.toml`` names no user of its own. - """ - recorded = HUD_ROOT / "image-user" - with contextlib.suppress(OSError): - return recorded.read_text(encoding="utf-8").strip() or None - return None - - -def _declared_directives(task_dir: Path, source_user: str | None) -> str: - """The task's declared environment, working dir and user as Dockerfile - directives. - - Docker applies these to every process in the container — the agent's - shells, the verifier, the serving process — which is the semantics Harbor - describes. Expressing them here rather than in the serving code keeps one - implementation (the container runtime's) instead of a second, partial one. - - Values are JSON-quoted, which is how Dockerfile reads a quoted operand - (shell quoting would bake the quote characters into the value), and the - task config is untrusted input, so anything that could open a new - directive is refused rather than interpolated. - - *source_user* is the env Dockerfile's own final ``USER``: the layer - installs as root, so the image's declared identity must be restored or - the adaptation would silently grant root where Harbor withheld it. - """ - policy = workspace_policy(task_dir) - lines = [] - for key, value in sorted(policy["env"].items()): +def _declared_directives(task: HarborTask, source_user: str | None) -> str: + """Render the task's container-wide environment directives.""" + lines: list[str] = [] + for key, value in sorted(task.config.environment.env.items()): if not _DOCKER_ENV_KEY.fullmatch(key): raise ValueError(f"environment.env key {key!r} is not a usable variable name") - # Docker substitutes ``$VAR`` in an ENV operand; the task declared a - # literal, so the dollar is escaped rather than expanded at build. literal = json.dumps(value).replace("$", "\\$") lines.append(f"ENV {key}={literal}") - if policy["workdir"]: - lines.append(f"WORKDIR {json.dumps(policy['workdir'])}") - # Validated at build, where a bad task config should fail, but applied per - # phase at runtime rather than as a USER directive — see :func:`phase_uid`. + if task.config.environment.workdir: + lines.append(f"WORKDIR {json.dumps(task.config.environment.workdir)}") for role in ("agent", "verifier"): - _validated_user(policy[f"{role}_user"], source_user) + _validated_user(task.config.phase_user(role), source_user) return "".join(f"{line}\n" for line in lines) @@ -299,74 +124,62 @@ async def adapt( path: str | Path, *, push: str | None = None, - build: bool = True, hud_requirement: str = "hud", -) -> dict[str, str]: - """Write adapted build contexts for every env group; build and push them. - - Returns ``{env_name: image_ref}`` — pass it to - :func:`~harbor.load` as ``images=`` so the rows carry - their image. *push* is a registry prefix (``registry.io/acme``); without it - images stay local, which serves ``DockerRuntime`` but not cloud - placements. ``build=False`` writes the contexts under ``.hud-adapt/`` and - stops. *hud_requirement* pins the hud installed in-image — a PyPI - requirement, or a path to a local wheel (baked into the context) for - unreleased SDKs; it must speak the same control-channel protocol as the - SDK driving the run. - """ +) -> Taskset: + """Build adapted images and return the runnable Harbor taskset.""" root = Path(path).resolve() out_root = root / ".hud-adapt" images: dict[str, str] = {} - for env_name, group_dirs in grouped(root): - context = _write_context(out_root / env_name, env_name, group_dirs, hud_requirement) - if not build: - continue + for env_name, group in _task_groups(root): + context = _write_context(out_root / env_name, env_name, group, hud_requirement) content = hash_directory(context) ref = f"{push}/{env_name}:{content}" if push else f"hud-harbor-adapted:{env_name}-{content}" deadlines = [ - t for t in (TaskConfig.read(d).environment.build_timeout_sec for d in group_dirs) if t + timeout for task in group if (timeout := task.config.environment.build_timeout_sec) ] await _docker( - "build", "--tag", ref, str(context), deadline=max(deadlines) if deadlines else None + "build", + "--tag", + ref, + str(context), + deadline=max(deadlines) if deadlines else None, ) if push: await _docker("push", ref) images[env_name] = ref - if images: LOGGER.info("adapted %d image(s)", len(images)) - return images + return _load(root, images=images) def _write_context( - context: Path, env_name: str, group_dirs: list[Path], hud_requirement: str + context: Path, env_name: str, group: list[HarborTask], hud_requirement: str ) -> Path: - """One group's adapted build context: env build context + the /hud layer.""" + """Write one group's environment and HUD adaptation layer.""" if context.exists(): shutil.rmtree(context) - env_dir = group_dirs[0] / "environment" + source = group[0] + env_dir = source.path / "environment" if not (env_dir / "Dockerfile").is_file(): raise FileNotFoundError(f"group {env_name!r} has no environment/Dockerfile") _copy_task_content(env_dir, context) dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - multi_step = [d.name for d in group_dirs if not (d / "instruction.md").is_file()] + multi_step = [task.path.name for task in group if not (task.path / "instruction.md").is_file()] if multi_step: raise NotImplementedError( "multi-step Harbor tasks (no root instruction.md) cannot be adapted yet: " + ", ".join(sorted(multi_step)[:5]) ) - for task_dir in group_dirs: - if reasons := unsupported_features(task_dir): + for task in group: + if reasons := task.unsupported_features(): raise NotImplementedError( - f"Harbor task {task_dir.name!r} declares behaviour this integration " + f"Harbor task {task.path.name!r} declares behaviour this integration " f"cannot reproduce: {'; '.join(reasons)}" ) dockerignore = context / ".dockerignore" if dockerignore.is_file(): - # The task's ignore rules were written for its own build; they must - # not exclude the adaptation layer from this one. _write( dockerignore, dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n!_hud_harbor\n!_hud_harbor/**\n", @@ -384,31 +197,27 @@ def _write_context( context / "_hud_harbor", ignore=shutil.ignore_patterns("tests", ".hud-adapt", *_CONTEXT_IGNORE_NAMES), ) - for task_dir in group_dirs: - target = hud_dir / "tasks" / task_dir.name + for task in group: + target = hud_dir / "tasks" / task.path.name target.mkdir(parents=True) for entry in ("instruction.md", "task.toml"): - _copy_task_content(task_dir / entry, target / entry) - _copy_task_content(task_dir / "tests", target / "tests") + _copy_task_content(task.path / entry, target / entry) + _copy_task_content(task.path / "tests", target / "tests") _write( hud_dir / "install.sh", _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)) .replace("__PYTHON__", SERVING_PYTHON) .replace("__HUD_ROOT__", str(HUD_ROOT)), ) - # The image's own USER, kept for serve time: the build context is gone by - # then, and a task that declares no user of its own still runs its phases - # as whatever the image said. - if (image_user := final_stage(dockerfile).user) is not None: - _write(hud_dir / "image-user", f"{image_user}\n") + if source.final_stage.user is not None: + _write(hud_dir / "image-user", f"{source.final_stage.user}\n") layer = ( _LAYER.replace("__HUD_ROOT__", str(HUD_ROOT)) .replace("__ENV_NAME__", env_name) .replace("__PYTHON__", SERVING_PYTHON) .replace( - # One env serves one policy, so the group's tasks agree on these. "__DECLARED__", - _declared_directives(group_dirs[0], final_stage(dockerfile).user), + _declared_directives(source, source.final_stage.user), ) ) _write(context / "Dockerfile", dockerfile + layer) @@ -416,13 +225,7 @@ def _write_context( def _copy_task_content(source: Path, destination: Path) -> None: - """Copy a task's own files into *destination*. - - A dataset is untrusted input: links are copied as links rather than - followed, so a task cannot pull host content in through a symlink — into - a build context, or into the ``/tests`` a rollout serves. Callers own - what *destination* is; this owns what "copy a task's files" means. - """ + """Copy task content without following symlinks.""" if source.is_dir(): shutil.copytree(source, destination, symlinks=True, ignore=_CONTEXT_IGNORE) else: @@ -430,471 +233,4 @@ def _copy_task_content(source: Path, destination: Path) -> None: def _write(path: Path, text: str) -> None: - """LF endings: these files run in Linux containers, where ``\\r\\n`` - breaks shebangs and shell scripts.""" path.write_text(text, encoding="utf-8", newline="\n") - - -def docker_runtime(**kwargs: Any) -> DockerRuntime: - """A local placement for adapted images. - - An adapted image sandboxes inside itself — that is what keeps the baked - tests and the verdict away from the agent — so its container needs the - nested-namespace relaxation that plain images should not be given. - """ - kwargs.setdefault("nested_sandbox", True) - return DockerRuntime(**kwargs) - - -# ─── what an adapted image serves, from inside the container ──────────── - - -def _harness_out_of_view() -> tuple[Mount, ...]: - """Mounts that leave the harness's tree absent from a session's namespace. - - A tmpfs over the tree itself would hide what is in it and still leave the - directory — and a directory named after the harness, at a path the task's - own image never had, is the one thing left telling the graded party what - is running it. Rebuilding the parent removes the entry instead: a fresh - tmpfs, then every sibling bound back as it was. Only ``--tmpfs`` and - ``--bind``, so it holds on the bubblewrap that ships with bullseye. - - What the sandbox loses is entries created *directly* in that parent during - the rollout, which is why it is one nothing writes to: writes anywhere - else, including new directories at the filesystem root, still land in the - real filesystem for the verifier to grade. - """ - parent = HUD_ROOT.parent - mounts = [Mount("tmpfs", dst=str(parent))] - if parent.is_dir(): - mounts += [ - Mount("rw", src=str(sibling), dst=str(sibling)) - for sibling in sorted(parent.iterdir()) - if sibling != HUD_ROOT - ] - return tuple(mounts) - - -def environment(ref: str | Path = HUD_ROOT / "tasks", *, name: str | None = None) -> Environment: - """The live env serving the task dirs under *ref* — the contract verb. - - Harbor environments are container filesystems, so this constructor is - meaningful only where the tasks' world is the current filesystem: inside - an image :func:`adapt` built, whose CMD serves exactly this. The - workspace is wherever the serving process starts (the image's - ``WORKDIR`` — the adaptation layer preserves it); grading runs each - task's ``tests/test.sh`` there, writing the Harbor reward under - ``/logs``. - """ - root = Path(ref) - task_dirs = sorted(d for d in root.iterdir() if d.is_dir()) - if not task_dirs: - raise ValueError(f"no Harbor tasks under {root}") - - # The image already carries the task's declared environment, working dir - # and user as Dockerfile directives, so this process starts inside them - # and sessions inherit them. - workdir = Path.cwd() - # An image with no WORKDIR starts at the filesystem root. The agent still - # gets the whole container (the bind below), but tracking every file in - # it is not a meaningful diff — and walking it stalls startup — so file - # tracking is off unless the task works somewhere specific. - rooted_at_filesystem = workdir == Path("/") - if rooted_at_filesystem: - LOGGER.warning( - "%s has no WORKDIR and declares no [environment] workdir; serving from / " - "with file tracking disabled", - root, - ) - policy = workspace_policy(task_dirs[0]) - - env = Environment(name or slugify(root.name)) - # The task's world is the whole container filesystem, exposed writable at - # its real paths — the sandbox is here to control the network namespace - # and to keep the graded material (baked tests, the serving venv) out of - # the graded party's reach, not to narrow the filesystem. - workspace = env.workspace( - workdir, - guest_path=workdir.as_posix(), - system_mounts=( - Mount("rw", src="/", dst="/"), - Mount("proc", dst="/proc"), - Mount("dev", dst="/dev"), - ), - # Masks go in ``mounts``, which bwrap applies *after* the workspace - # bind — as system mounts they would be re-covered when the guest - # path is ``/`` (an image with no WORKDIR). ``/tests`` and the verdict - # are handled by not existing during the agent phase at all - # (:func:`_hide_grading_dirs`), which is what Harbor does; the - # harness's own tree has to exist throughout, being what serves, so it - # is hidden by rebuilding its parent instead. - mounts=_harness_out_of_view(), - credentials_dir=HUD_ROOT / "session-keys", - # The agent's declared identity, not the harness's: the serving - # process stays root so it can place and remove the grading - # directories and keep its own state under /hud. - shell_uid=(agent_uid := phase_uid(task_dirs[0], "agent")), - # Whose the workspace is, is the image's statement — Harbor does not - # re-own it for the agent, so neither does this. - hand_over_root=False, - track_files=False if rooted_at_filesystem else None, - # The agent phase's own variables, scoped to its sessions. A dropped - # session would otherwise be pointed at the harness's home. - env={**({"HOME": home} if (home := phase_home(agent_uid)) else {}), **policy["agent_env"]}, - network=policy["network"], - # What the task declared its agent may reach. Given a set, the - # workspace takes a network of its own whose only way out applies it — - # which is also what puts the substrate, and the channel grading this - # rollout, out of the agent's reach. - allowed_hosts=policy["allowed_hosts"], - # Always: the sandbox is what keeps the baked tests and the serving - # venv out of the graded party's reach — an unsandboxed fallback - # would hand the agent the verifier's answers. - require_isolation=True, - ) - - for task_dir in task_dirs: - _register(env, task_dir, workdir, workspace) - return env - - -def _register(env: Environment, task_dir: Path, workdir: Path, workspace: Workspace) -> None: - config = TaskConfig.read(task_dir) - - @env.template( - id=task_dir.name, - description=config.task.description or f"Harbor task {task_dir.name}", - ) - async def _run_harbor_task() -> AsyncGenerator[Any, Any]: - # Harbor uploads /tests when it runs the verifier, so its agent phase - # never contains the assertions it is graded on — nor the verdict dir. - # Adapted images ship both (empty), and an earlier rollout in this - # container leaves them behind, so the agent phase starts by removing - # them and ends the same way: :func:`_grade` lays them down in between, - # for as long as the verifier needs them. - _hide_grading_dirs() - try: - answer = yield (task_dir / "instruction.md").read_text(encoding="utf-8") - yield await _grade(task_dir, workdir, answer, workspace) - finally: - _hide_grading_dirs() - # Harbor's agent phase is one continuous session, so a service the - # agent starts is still running when the verifier looks for it — - # and is gone before the next rollout, which this container may - # well serve too. - await workspace.discard_sandbox() - - -def _sync_tests(task_dir: Path) -> None: - """Leave ``/tests`` holding exactly *task_dir*'s tests.""" - _reset_dir(TESTS) - for child in (task_dir / "tests").iterdir(): - _copy_task_content(child, TESTS / child.name) - - -def _hide_grading_dirs() -> None: - """Leave no ``/tests`` or verdict directory for the agent phase to find. - - Removed rather than emptied: an empty directory the base image does not - have is itself a signal, and these two are the harness's whole visible - footprint outside its own tree. Where the serve process cannot remove them - — not root, and they sit at the filesystem root — emptying is the fallback, - which is what the mask used to achieve. - """ - for path in (TESTS, VERIFIER_LOGS): - try: - shutil.rmtree(path) - except FileNotFoundError: - continue - except OSError: - LOGGER.debug("could not remove %s; emptying it instead", path) - _reset_dir(path) - - -def _reset_dir(path: Path) -> None: - """Leave *path* an existing, empty directory, whatever it was before. - - Contents are cleared in place rather than removing and recreating the - directory: a non-root serve process can empty ``/tests`` but cannot - recreate it at the filesystem root. - """ - if path.is_symlink() or path.is_file(): - path.unlink() - if not path.is_dir(): - path.mkdir(parents=True) - return - for child in path.iterdir(): - if child.is_symlink() or child.is_file(): - child.unlink() - else: - shutil.rmtree(child) - - -async def _grade( - task_dir: Path, workdir: Path, answer: Any, workspace: Workspace | None = None -) -> dict[str, Any]: - logs = LOGS - # The agent shares the container, so restore the verifier from the baked - # (masked) copy before running it. - _sync_tests(task_dir) - - # Run the script itself: its shebang picks the interpreter Harbor's task - # intends, and a minimal image may have no bash at all. - test_sh = TESTS / "test.sh" - test_sh.chmod(test_sh.stat().st_mode | 0o111) - argv = [str(test_sh)] - - config = TaskConfig.read(task_dir) - verifier_env: dict[str, str] = {} - verifier_uid = phase_uid(task_dir, "verifier") - setpriv = shutil.which("setpriv") - if verifier_uid is not None and setpriv is not None and os.geteuid() == 0: - # Root laid the grading directories down, so hand them over before - # dropping: a verifier that cannot write /logs/verifier cannot produce - # the verdict it is being run to produce. The verdict directory is made - # here rather than by the grading helper, since it must exist first. - _reset_dir(VERIFIER_LOGS) - for root_path in (TESTS, VERIFIER_LOGS): - for target in (root_path, *root_path.rglob("*")): - os.lchown(target, verifier_uid, verifier_uid) - if home := phase_home(verifier_uid): - verifier_env["HOME"] = home - argv = [ - setpriv, - "--reuid", - str(verifier_uid), - "--regid", - str(verifier_uid), - "--clear-groups", - *argv, - ] - severed = not config.network("verifier") - if severed and (bwrap := usable_bwrap()) is None: - raise RuntimeError( - "the verifier declares no-network but bwrap cannot sandbox here; " - "refusing to grade with network access the task ruled out" - ) - - async def run_tests() -> ProcessGroup: - spawn = argv - environ = {**os.environ, **verifier_env, **config.verifier.env} - if not severed: - return await create_process_group_exec( - *spawn, - cwd=workdir, - env=environ, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - # A network namespace of its own is the only reason the verifier is - # sandboxed at all, and bwrap can only make one inside a user - # namespace. That namespace maps a single id unless this side says - # otherwise — and a verifier that cannot chown is one that cannot - # untar, or run apt, which is most of what Harbor's verifiers do - # before they assert anything. So it is held at creation and mapped, - # exactly as an agent session's sandbox is. - info_read, info_write = os.pipe() - block_read, block_write = os.pipe() - try: - os.set_inheritable(info_write, True) - os.set_inheritable(block_read, True) - # /dev and /proc are fresh: binding the real root inside a user - # namespace leaves device nodes unwritable, and a test.sh - # redirecting to /dev/null would fail on that alone. - group = await create_process_group_exec( - bwrap, - "--unshare-user", - "--info-fd", - str(info_write), - "--userns-block-fd", - str(block_read), - "--bind", - "/", - "/", - "--dev", - "/dev", - "--proc", - "/proc", - "--unshare-net", - "--", - *spawn, - cwd=workdir, - env=environ, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - pass_fds=(info_write, block_read), - ) - os.close(info_write) - os.close(block_read) - info_write = block_read = -1 - await install_identity_map(info_read, block_write) - return group - finally: - os.close(info_read) - os.close(block_write) - for stray in (info_write, block_read): - if stray != -1: - os.close(stray) - - if workspace is None or severed or not workspace.owns_netns: - return await _grade_with_verifier(config, logs, answer, run_tests) - sandbox = await workspace.sandbox_pid() - if sandbox is None: - return await _grade_with_verifier(config, logs, answer, run_tests) - - # Harbor grades a service by talking to it — telnet to the VM the agent - # booted, git clone from the server it configured. Those listen in the - # workspace's network now, so the verifier is run there: its user and - # network namespaces, and nothing else. Keeping this mount namespace is the - # point — /tests and the verdict are here, where the workspace cannot reach - # them — and the ids map through, so it stays able to chown what it untars. - async with workspace.visiting(config.allowed_hosts("verifier")) as visitor_env: - # Behind the workspace's boundary now, so it needs a way out — one of - # its own, since Harbor's verifiers commonly install their own tooling - # first and the hosts they may reach are the ones the *verifier* - # declared, not the ones the agent was held to. - verifier_env.update(visitor_env) - nsenter = shutil.which("nsenter") or "/usr/bin/nsenter" - argv = [ - nsenter, - "--target", - str(sandbox), - "--user", - "--net", - "--preserve-credentials", - "--", - *argv, - ] - return await _grade_with_verifier(config, logs, answer, run_tests) - - -# ─── verifier grading and docker plumbing ─────────────────────────────── - - -async def _grade_with_verifier( - config: TaskConfig, - logs: Path, - answer: Any, - run_tests: Callable[[], Awaitable[ProcessGroup]], -) -> dict[str, Any]: - """Run the Harbor verifier and shape its output into a HUD grade. - - *run_tests* starts the ``tests/test.sh`` process group wherever it must - run; this owns the answer file, the ``[verifier] timeout_sec`` bound, and - parsing ``reward.json``/``reward.txt``. - """ - timeout = config.verifier.timeout_sec or DEFAULT_VERIFIER_TIMEOUT - # Harbor's harness guarantees the verifier output dir exists. The agent's - # namespace masks it, but this process shares the container with whatever - # the *setup* left behind, so the verdict dir is recreated and the answer - # file is written without following anything into it. - _reset_dir(logs / "verifier") - _write_no_follow(logs / "agent_answer.txt", "" if answer is None else str(answer)) - group = await run_tests() - proc = group.process - # Drain while waiting, and bound on the script's own exit. Draining only - # after exit deadlocks a chatty verifier once the pipe buffer fills; - # waiting for pipe EOF instead reports a timeout for a script that - # finished but left a daemon holding its stdout. Reading concurrently and - # timing the process avoids both. - assert proc.stdout is not None and proc.stderr is not None - reading = ( - asyncio.create_task(proc.stdout.read()), - asyncio.create_task(proc.stderr.read()), - ) - try: - try: - await asyncio.wait_for(proc.wait(), timeout=timeout) - timed_out = False - except TimeoutError: - timed_out = True - # The group is the boundary however test.sh ends: descendants release - # the inherited pipes here (so the reads complete), and none survive - # to write /logs during the next rollout's grading. - await group.terminate() - out_bytes, err_bytes = await asyncio.gather(*reading) - except BaseException: - # A cancelled rollout unwinds through here; the group and the readers - # are this function's to release however it exits. - for reader in reading: - reader.cancel() - await group.terminate() - raise - if timed_out: - return { - "score": 0.0, - "isError": True, - "content": f"Harbor verifier timed out after {timeout:.0f}s", - "info": { - "verifier_timeout_sec": timeout, - "stdout": out_bytes.decode("utf-8", "replace")[-4000:], - "stderr": err_bytes.decode("utf-8", "replace")[-4000:], - }, - } - # The reward file is Harbor's verdict, not the exit code: verifiers - # commonly end with `exit "$status"`, carrying the test suite's code after - # writing the score it implies. A verifier that exits nonzero *without* - # writing a reward is caught below, as a grading error. - reward, info = _read_reward(logs / "verifier") - info.update( - { - "stdout": out_bytes.decode("utf-8", "replace")[-4000:], - "stderr": err_bytes.decode("utf-8", "replace")[-4000:], - } - ) - if reward is None: - return { - "score": 0.0, - "isError": True, - "content": "Harbor verifier did not write reward.json or reward.txt", - "info": info, - } - return {"score": reward, "info": info} - - -def _read_reward(verifier_logs: Path) -> tuple[float | None, dict[str, Any]]: - """Parse Harbor's verifier output: ``reward.json`` first, then ``reward.txt``. - - A reward is a finite number; booleans (an ``int`` subclass) and - ``nan``/``inf`` are parse failures, not scores. - """ - reward_json = verifier_logs / "reward.json" - if reward_json.is_file(): - try: - data = json.loads(reward_json.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return None, {"parse_error": "reward.json is not valid JSON"} - if (score := _as_score(data)) is not None: - return score, {"reward_file": str(reward_json)} - if isinstance(data, dict): - for key in ("reward", "score"): - if (score := _as_score(data.get(key))) is not None: - return score, {"reward_file": str(reward_json), "reward_json": data} - return None, {"reward_file": str(reward_json), "reward_parse_error": "no numeric reward"} - - reward_txt = verifier_logs / "reward.txt" - if reward_txt.is_file(): - text = reward_txt.read_text(encoding="utf-8").strip() - try: - value = float(text) - except ValueError: - return None, {"reward_file": str(reward_txt), "reward_parse_error": text} - if not math.isfinite(value): - return None, {"reward_file": str(reward_txt), "reward_parse_error": text} - return value, {"reward_file": str(reward_txt)} - - return None, {} - - -def _write_no_follow(path: Path, text: str) -> None: - """Write *text* to *path* itself — never through a symlink planted there.""" - if path.is_symlink(): - path.unlink() - with path.open("w", encoding="utf-8") as handle: - handle.write(text) - - -def _as_score(value: Any) -> float | None: - if isinstance(value, bool) or not isinstance(value, int | float): - return None - return float(value) if math.isfinite(float(value)) else None diff --git a/integrations/harbor/_export.py b/integrations/harbor/_export.py index b5c2f9c8c..93cb97661 100644 --- a/integrations/harbor/_export.py +++ b/integrations/harbor/_export.py @@ -121,24 +121,7 @@ def _resolve_env(task: Task, authored: dict[str, Environment]) -> Environment: hud serve {serve_target} --port {port} & -# Wait for the control channel to accept connections (python is always present). -# A container that never serves, or a task that will not start, is broken -# infrastructure — failing here is honest, where continuing would let the -# verifier score the run 0 as though the agent had simply failed. -python3 -c 'import socket, sys, time -port = int(sys.argv[1]) -for _ in range(120): - try: - socket.create_connection(("127.0.0.1", port), 0.5).close() - sys.exit(0) - except OSError: - time.sleep(0.5) -sys.exit(1)' {port} || {{ - echo "hud: control channel never came up on port {port}" >&2 - exit 1 -}} - -# Run the task setup phase and park the run for grading. +# The SDK connection performs protocol-level readiness retries before setup. hud task start {task} --args {args_json} --url tcp://127.0.0.1:{port} || {{ echo "hud: task setup failed; refusing to run the agent against an unset task" >&2 exit 1 diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py index b940d1288..465a65a9a 100644 --- a/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -117,7 +117,11 @@ def read(cls, task_dir: Path) -> TaskConfig: ) from error def phase(self, role: str) -> _Phase: - return self.agent if role == "agent" else self.verifier + if role == "agent": + return self.agent + if role == "verifier": + return self.verifier + raise ValueError(f"unknown Harbor phase {role!r}") def network(self, role: str) -> bool: """Whether *role*'s processes may reach the network at all. @@ -160,181 +164,46 @@ def phase_user(self, role: str) -> str | int | None: """ return self.phase(role).user - @property - def user(self) -> str | int | None: - """The identity the task's phases run as, if either names one.""" - return self.agent.user if self.agent.user is not None else self.verifier.user - def detect(path: str | Path) -> bool: """True when *path* is a Harbor task dir or a dataset of them.""" return bool(task_dirs(path)) -def load(path: str | Path, *, images: dict[str, str] | None = None) -> Taskset: +def load(path: str | Path) -> Taskset: """Load a Harbor task dir (or dataset dir) into a :class:`Taskset`. One row per task dir (``id`` = the dir name); rows share one env name per - distinct environment (see :func:`grouped` for what distinguishes them and - how the name is derived). Each row carries the task's declared launch - requirements (:func:`runtime_config`: cpu/memory/gpu and time budgets), - plus the adapted image ref once - :func:`~harbor.adapt` has produced it, so it runs on any - container placement:: - - await harbor.adapt(path) - job = await harbor.load(path).run(agent, runtime=DockerRuntime()) + distinct environment. Each row carries the task's declared launch + requirements (cpu/memory/gpu). Adapted images are bound by :func:`adapt` + before the returned taskset is run:: + + taskset = await harbor.adapt(path) + job = await taskset.run(agent, runtime=DockerRuntime()) """ + return _load(path) + + +def _load(path: str | Path, *, images: dict[str, str] | None = None) -> Taskset: + """Load rows, optionally binding image refs produced by :func:`adapt`.""" root = Path(path).resolve() dataset_name = root.parent.name if is_harbor_task(root) else root.name - if not task_dirs(root): - raise ValueError(f"no Harbor tasks found in {path}") tasks: list[Task] = [] - for env_name, group_dirs in grouped(root): + for env_name, group in _task_groups(root): image = (images or {}).get(env_name) tasks.extend( Task( env=env_name, - id=task_dir.name, - columns=columns(task_dir), - runtime_config=runtime_config(task_dir, image=image), + id=task.path.name, + columns=task.columns, + runtime_config=task.runtime_config(image=image), ) - for task_dir in group_dirs + for task in group ) return Taskset(slugify(dataset_name), tasks, origin=f"harbor:{root}") -def columns(task_dir: Path) -> dict[str, Any] | None: - """The task's ``[metadata]`` (plus keywords) as filterable columns.""" - config = TaskConfig.read(task_dir) - fields = dict(config.metadata) - if config.task.keywords: - fields.setdefault("keywords", config.task.keywords) - return fields or None - - -def workspace_policy(task_dir: Path) -> dict[str, Any]: - """What the task declares about the workspace its agent works in. - - Grouping keys on this, so tasks that mean different things never share - an environment. Only settings this integration can honor appear here — - see :func:`unsupported_features` for the rest. - """ - config = TaskConfig.read(task_dir) - return { - "network": config.network("agent"), - # Container-wide variables reach every process (baked as image ENV); - # the agent phase's reach only its sessions, and the verifier's are - # applied where the verifier runs. Each phase sees what Harbor scoped - # to it, and all three are in this key so tasks that differ never - # share an environment. - "env": dict(config.environment.env), - "agent_env": dict(config.agent.env), - "workdir": config.environment.workdir or None, - "user": config.user, - # Sorted rather than a set: the policy is hashed to key environments, - # so it has to serialize, and two tasks naming the same hosts in a - # different order declare the same thing. - "allowed_hosts": sorted(config.allowed_hosts("agent")), - # The verifier's are its own. Grouped on as well, since a task whose - # grader may reach different hosts is a different environment. - "verifier_allowed_hosts": sorted(config.allowed_hosts("verifier")), - "agent_user": config.phase_user("agent"), - "verifier_user": config.phase_user("verifier"), - } - - -def runtime_config(task_dir: Path, *, image: str | None = None) -> RuntimeConfig | None: - """The task's declared launch requirements as HUD's portable config. - - ``storage_mb`` has no portable counterpart and is dropped; time budgets - bound the *rollout*, not the substrate, so they stay out of here (see - :func:`agent_timeout`). - """ - environment = TaskConfig.read(task_dir).environment - resources = RuntimeResources( - cpu=environment.cpus or None, - memory_mb=environment.memory_mb or None, - gpu=RuntimeGPU( - count=environment.gpus, - type=next((t for t in environment.gpu_types if t), None), - ) - if environment.gpus - else None, - ) - declared = RuntimeConfig( - image=image, - resources=resources if resources.model_dump(exclude_none=True) else None, - ) - return declared if declared.model_dump(exclude_none=True) else None - - -def agent_timeout(task_dir: Path) -> float | None: - """How long the task allows the agent to work (``[agent] timeout_sec``). - - A rollout budget, not a launch requirement: pass it as ``rollout_timeout`` - when running the row. - """ - return TaskConfig.read(task_dir).agent.timeout_sec or None - - -def unsupported_features(task_dir: Path) -> list[str]: - """Declarations this integration cannot reproduce faithfully. - - A wrong score is worse than a refused task, so each of these names itself - rather than being silently dropped. - """ - config = TaskConfig.read(task_dir) - environment, agent, verifier = config.environment, config.agent, config.verifier - reasons: list[str] = [] - - if environment.os not in (None, "linux"): - reasons.append(f"environment.os={environment.os!r}") - if environment.tpu: - reasons.append("environment.tpu (no TPU resource model)") - if agent.user is not None and verifier.user is not None and agent.user != verifier.user: - # One image, one USER. Only one phase naming an identity is fine — - # both phases run as it; two *different* identities are not. - reasons.append("agent.user and verifier.user differ (the image has one USER)") - workdir = environment.workdir or _final_stage_workdir(task_dir) - if workdir and Path(workdir).is_relative_to(HUD_ROOT): - # The adaptation layer owns that tree inside the image and hides it from - # agent sessions; a task working there would find it empty. - reasons.append(f"working directory {workdir!r} is inside {HUD_ROOT} (reserved)") - if environment.docker_image and not (task_dir / "environment" / "Dockerfile").is_file(): - reasons.append( - "prebuilt docker_image environments (adapt builds from environment/Dockerfile)" - ) - - # Everything below describes the container's own boot process, which - # adaptation replaces with the serving CMD: services an ENTRYPOINT would - # start never start, so healthchecks would await nothing and MCP server - # URLs would point at nothing. - dockerfile = task_dir / "environment" / "Dockerfile" - directives = ( - final_stage(dockerfile.read_text("utf-8", errors="replace")).directives - if dockerfile.is_file() - else frozenset() - ) - if "ENTRYPOINT" in directives: - reasons.append("environment/Dockerfile ENTRYPOINT (adaptation replaces container startup)") - if any( - (task_dir / "environment" / name).is_file() - for name in ("docker-compose.yaml", "docker-compose.yml", "compose.yaml", "compose.yml") - ): - reasons.append("compose environments (sidecar services would never start)") - if environment.healthcheck: - reasons.append("environment.healthcheck (nothing starts the services it would await)") - if environment.mcp_servers: - reasons.append("environment.mcp_servers (nothing starts the servers they point at)") - if verifier.environment_mode or verifier.environment: - reasons.append("verifier runs in its own environment") - if config.steps: - reasons.append("multi-step tasks ([[steps]])") - return reasons - - @dataclass(frozen=True) class FinalStage: """What a Dockerfile's last stage declares. @@ -348,6 +217,7 @@ class FinalStage: directives: frozenset[str] = frozenset() user: str | None = None + workdir: str | None = None def dockerfile_instructions(dockerfile_text: str) -> list[tuple[str, str, list[int]]]: @@ -399,77 +269,168 @@ def dockerfile_instructions(dockerfile_text: str) -> list[tuple[str, str, list[i return instructions -def _final_stage_workdir(task_dir: Path) -> str | None: - """The ``WORKDIR`` the image's final stage ends in, if it sets one.""" - dockerfile = task_dir / "environment" / "Dockerfile" - if not dockerfile.is_file(): - return None - workdir = None - for word, operand, _ in dockerfile_instructions( - dockerfile.read_text("utf-8", errors="replace") - ): - if word == "FROM": - workdir = None - elif word == "WORKDIR": - workdir = operand.strip().strip('"') or None - return workdir - - def final_stage(dockerfile_text: str) -> FinalStage: """Parse *dockerfile_text* into its :class:`FinalStage`.""" directives: set[str] = set() user: str | None = None + workdir: str | None = None for word, operand, _ in dockerfile_instructions(dockerfile_text): if word == "FROM": - directives, user = set(), None + directives, user, workdir = set(), None, None elif word == "USER": user = operand or None + elif word == "WORKDIR": + workdir = operand.strip().strip('"') or None directives.add(word) return FinalStage( - frozenset(directives), None if user in ("root", "0", "root:root", "0:0") else user + frozenset(directives), + None if user in ("root", "0", "root:root", "0:0") else user, + workdir, ) -def grouped(root: str | Path) -> list[tuple[str, list[Path]]]: - """Task dirs grouped by the env they need, under content-derived names. +@dataclass(frozen=True, slots=True) +class HarborTask: + """The parsed, immutable view of one Harbor task directory. - One env name per group — ``-`` over everything that - decides what the group's image is — and that name is the join key - between :func:`load`'s rows and :func:`~harbor.adapt`'s images. Deriving - it from content rather than position is what makes the join safe: a name - denotes one image forever, so editing, adding or removing a task can - never leave a row pointing at an environment that has since come to mean - something else. + Loading, grouping, image adaptation, and runtime setup all consume this + same record. That keeps task.toml and Dockerfile parsing at one boundary + instead of making each phase rediscover the task from its path. """ + + path: Path + config: TaskConfig + dockerfile: str + final_stage: FinalStage + environment_hash: str + + @classmethod + def read(cls, task_dir: Path) -> HarborTask: + path = Path(task_dir).resolve() + dockerfile_path = path / "environment" / "Dockerfile" + dockerfile = ( + dockerfile_path.read_text("utf-8", errors="replace") + if dockerfile_path.is_file() + else "" + ) + environment = path / "environment" + return cls( + path=path, + config=TaskConfig.read(path), + dockerfile=dockerfile, + final_stage=final_stage(dockerfile), + environment_hash=hash_directory(environment) if environment.exists() else "no-env", + ) + + @property + def columns(self) -> dict[str, Any] | None: + fields = dict(self.config.metadata) + if self.config.task.keywords: + fields.setdefault("keywords", self.config.task.keywords) + return fields or None + + def runtime_config(self, *, image: str | None = None) -> RuntimeConfig | None: + """Map portable launch requirements into the SDK's runtime config.""" + environment = self.config.environment + resources = RuntimeResources( + cpu=environment.cpus or None, + memory_mb=environment.memory_mb or None, + gpu=RuntimeGPU( + count=environment.gpus, + type=next((gpu_type for gpu_type in environment.gpu_types if gpu_type), None), + ) + if environment.gpus + else None, + ) + declared = RuntimeConfig( + image=image, + resources=resources if resources.model_dump(exclude_none=True) else None, + ) + return declared if declared.model_dump(exclude_none=True) else None + + @property + def workspace_key(self) -> str: + """Stable serialization of the workspace contract used for grouping.""" + config = self.config + return json.dumps( + { + "network": config.network("agent"), + "env": dict(config.environment.env), + "agent_env": dict(config.agent.env), + "workdir": config.environment.workdir or None, + "user": ( + config.agent.user if config.agent.user is not None else config.verifier.user + ), + "allowed_hosts": sorted(config.allowed_hosts("agent")), + "verifier_allowed_hosts": sorted(config.allowed_hosts("verifier")), + "agent_user": config.phase_user("agent"), + "verifier_user": config.phase_user("verifier"), + }, + sort_keys=True, + ) + + def unsupported_features(self) -> list[str]: + """Declarations this integration cannot reproduce faithfully.""" + config = self.config + environment, agent, verifier = config.environment, config.agent, config.verifier + reasons: list[str] = [] + + if environment.os not in (None, "linux"): + reasons.append(f"environment.os={environment.os!r}") + if environment.tpu: + reasons.append("environment.tpu (no TPU resource model)") + if agent.user is not None and verifier.user is not None and agent.user != verifier.user: + reasons.append("agent.user and verifier.user differ (the image has one USER)") + workdir = environment.workdir or self.final_stage.workdir + if workdir and Path(workdir).is_relative_to(HUD_ROOT): + reasons.append(f"working directory {workdir!r} is inside {HUD_ROOT} (reserved)") + if environment.docker_image and not (self.path / "environment" / "Dockerfile").is_file(): + reasons.append( + "prebuilt docker_image environments (adapt builds from environment/Dockerfile)" + ) + if "ENTRYPOINT" in self.final_stage.directives: + reasons.append( + "environment/Dockerfile ENTRYPOINT (adaptation replaces container startup)" + ) + if any( + (self.path / "environment" / name).is_file() + for name in ("docker-compose.yaml", "docker-compose.yml", "compose.yaml", "compose.yml") + ): + reasons.append("compose environments (sidecar services would never start)") + if environment.healthcheck: + reasons.append("environment.healthcheck (nothing starts the services it would await)") + if environment.mcp_servers: + reasons.append("environment.mcp_servers (nothing starts the servers they point at)") + if verifier.environment_mode or verifier.environment: + reasons.append("verifier runs in its own environment") + if config.steps: + reasons.append("multi-step tasks ([[steps]])") + return reasons + + +def _task_groups(root: str | Path) -> list[tuple[str, list[HarborTask]]]: resolved = Path(root).resolve() dataset_name = resolved.parent.name if is_harbor_task(resolved) else resolved.name dirs = task_dirs(resolved) if not dirs: raise ValueError(f"no Harbor tasks found in {root}") - groups: dict[tuple[str, str], list[Path]] = {} - for task_dir in dirs: - env_dir = task_dir / "environment" - env_hash = hash_directory(env_dir) if env_dir.exists() else "no-env" - # Tasks sharing a build context still need separate envs when their - # declared workspace behaviour differs: one env serves one policy. - # Invariant: everything environment() consumes per group is either in - # this key (workspace_policy) or refused (unsupported_features) — a - # declaration outside both would silently take the first task's value - # for the whole group. - policy = json.dumps(workspace_policy(task_dir), sort_keys=True) - groups.setdefault((env_hash, policy), []).append(task_dir) + groups: dict[tuple[str, str], list[HarborTask]] = {} + for task in (HarborTask.read(task_dir) for task_dir in dirs): + groups.setdefault((task.environment_hash, task.workspace_key), []).append(task) base_name = slugify(dataset_name) return sorted( - (f"{base_name}-{_group_digest(env_hash, policy)}", group) - for (env_hash, policy), group in groups.items() + ( + f"{base_name}-{_group_digest(environment_hash, workspace_key)}", + group, + ) + for (environment_hash, workspace_key), group in groups.items() ) -def _group_digest(env_hash: str, policy: str) -> str: - """Short stable digest of one group's key — its build context and the - workspace policy its image bakes in, which together are the image.""" - return hashlib.sha256(f"{env_hash}\0{policy}".encode()).hexdigest()[:12] +def _group_digest(environment_hash: str, workspace_key: str) -> str: + """Short stable digest of one group's build and workspace contract.""" + return hashlib.sha256(f"{environment_hash}\0{workspace_key}".encode()).hexdigest()[:12] # ─── task-dir primitives ──────────────────────────────────────────────── diff --git a/integrations/harbor/_runtime.py b/integrations/harbor/_runtime.py new file mode 100644 index 000000000..3e5827693 --- /dev/null +++ b/integrations/harbor/_runtime.py @@ -0,0 +1,388 @@ +"""Runtime for an adapted Harbor image.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import math +import os +import pwd +import shutil +from collections.abc import AsyncGenerator, Awaitable, Callable # noqa: TC003 +from pathlib import Path +from typing import Any + +from hud.environment import Environment, Mount +from hud.environment.workspace import Workspace, install_identity_map +from hud.utils.process import ProcessGroup, create_process_group_exec + +from ._adapt import _copy_task_content, _validated_user +from ._load import DEFAULT_VERIFIER_TIMEOUT, HUD_ROOT, HarborTask, TaskConfig, slugify + +LOGGER = logging.getLogger(__name__) + +LOGS = Path("/logs") +VERIFIER_LOGS = LOGS / "verifier" +TESTS = Path("/tests") + + +def phase_uid(task: HarborTask | Path, role: str) -> int | None: + """Resolve the identity for one Harbor phase inside the adapted image.""" + if not isinstance(task, HarborTask): + task = HarborTask.read(task) + user = _validated_user(task.config.phase_user(role), _image_user()) + if user is None: + return None + name = user.split(":", 1)[0] + if name.isdigit(): + return int(name) + try: + return pwd.getpwnam(name).pw_uid + except KeyError as error: + raise ValueError(f"declared user {user!r} does not exist in this image") from error + + +def phase_home(uid: int | None) -> str | None: + """Return the home directory for a phase that drops to uid.""" + if uid is None: + return None + with contextlib.suppress(KeyError): + return pwd.getpwuid(uid).pw_dir + return None + + +def _image_user() -> str | None: + """Read the final image USER recorded by the adaptation layer.""" + with contextlib.suppress(OSError): + return (HUD_ROOT / "image-user").read_text(encoding="utf-8").strip() or None + return None + + +def _harness_out_of_view() -> tuple[Mount, ...]: + """Hide the adapted harness tree from agent sessions.""" + parent = HUD_ROOT.parent + mounts = [Mount("tmpfs", dst=str(parent))] + if parent.is_dir(): + mounts += [ + Mount("rw", src=str(sibling), dst=str(sibling)) + for sibling in sorted(parent.iterdir()) + if sibling != HUD_ROOT + ] + return tuple(mounts) + + +def environment(ref: str | Path = HUD_ROOT / "tasks", *, name: str | None = None) -> Environment: + """Serve the task directories baked into an adapted Harbor image.""" + root = Path(ref) + task_dirs = sorted(d for d in root.iterdir() if d.is_dir()) + if not task_dirs: + raise ValueError(f"no Harbor tasks under {root}") + + tasks = [HarborTask.read(task_dir) for task_dir in task_dirs] + source = tasks[0] + workdir = Path.cwd() + rooted_at_filesystem = workdir == Path("/") + if rooted_at_filesystem: + LOGGER.warning( + "%s has no WORKDIR and declares no [environment] workdir; serving from / " + "with file tracking disabled", + root, + ) + + agent_uid = phase_uid(source, "agent") + env = Environment(name or slugify(root.name)) + workspace = env.workspace( + workdir, + guest_path=workdir.as_posix(), + system_mounts=( + Mount("rw", src="/", dst="/"), + Mount("proc", dst="/proc"), + Mount("dev", dst="/dev"), + ), + mounts=_harness_out_of_view(), + credentials_dir=HUD_ROOT / "session-keys", + shell_uid=agent_uid, + hand_over_root=False, + track_files=False if rooted_at_filesystem else None, + env={ + **({"HOME": home} if (home := phase_home(agent_uid)) else {}), + **source.config.agent.env, + }, + network=source.config.network("agent"), + allowed_hosts=source.config.allowed_hosts("agent"), + require_isolation=True, + ) + + for task in tasks: + _register(env, task, workdir, workspace) + return env + + +def _register(env: Environment, task: HarborTask, workdir: Path, workspace: Workspace) -> None: + config = task.config + + @env.template( + id=task.path.name, + description=config.task.description or f"Harbor task {task.path.name}", + ) + async def _run_harbor_task() -> AsyncGenerator[Any, Any]: + _hide_grading_dirs() + try: + answer = yield (task.path / "instruction.md").read_text(encoding="utf-8") + yield await _grade(task, workdir, answer, workspace) + finally: + _hide_grading_dirs() + await workspace.discard_sandbox() + + +def _sync_tests(task: HarborTask) -> None: + """Leave /tests holding exactly the task's tests.""" + _reset_dir(TESTS) + for child in (task.path / "tests").iterdir(): + _copy_task_content(child, TESTS / child.name) + + +def _hide_grading_dirs() -> None: + """Remove the verifier's tests and verdict from the agent's view.""" + for path in (TESTS, VERIFIER_LOGS): + try: + shutil.rmtree(path) + except FileNotFoundError: + continue + except OSError: + LOGGER.debug("could not remove %s; emptying it instead", path) + _reset_dir(path) + + +def _reset_dir(path: Path) -> None: + """Leave path as an existing, empty directory.""" + if path.is_symlink() or path.is_file(): + path.unlink() + if not path.is_dir(): + path.mkdir(parents=True) + return + for child in path.iterdir(): + if child.is_symlink() or child.is_file(): + child.unlink() + else: + shutil.rmtree(child) + + +async def _grade( + task: HarborTask, workdir: Path, answer: Any, workspace: Workspace | None = None +) -> dict[str, Any]: + _sync_tests(task) + test_sh = TESTS / "test.sh" + test_sh.chmod(test_sh.stat().st_mode | 0o111) + command = [str(test_sh)] + verifier_command = command + + config = task.config + verifier_env: dict[str, str] = {} + verifier_uid = phase_uid(task, "verifier") + setpriv = shutil.which("setpriv") + if verifier_uid is not None and setpriv is not None and os.geteuid() == 0: + _reset_dir(VERIFIER_LOGS) + for root_path in (TESTS, VERIFIER_LOGS): + for target in (root_path, *root_path.rglob("*")): + os.lchown(target, verifier_uid, verifier_uid) + if home := phase_home(verifier_uid): + verifier_env["HOME"] = home + verifier_command = [ + setpriv, + "--reuid", + str(verifier_uid), + "--regid", + str(verifier_uid), + "--clear-groups", + *command, + ] + + severed = not config.network("verifier") + verifier_workspace = workspace + if severed and verifier_workspace is None: + verifier_workspace = Workspace( + Path("/"), + system_mounts=(Mount("proc", dst="/proc"), Mount("dev", dst="/dev")), + network=False, + allowed_hosts=(), + guest_path="/", + hand_over_root=False, + ) + if severed and (verifier_workspace is None or not verifier_workspace.bwrap_available): + raise RuntimeError( + "the verifier declares no-network but bwrap cannot sandbox here; " + "refusing to grade with network access the task ruled out" + ) + + spawn = verifier_command + + async def run_tests() -> ProcessGroup: + environ = {**os.environ, **verifier_env, **config.verifier.env} + if not severed: + return await create_process_group_exec( + *spawn, + cwd=workdir, + env=environ, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert verifier_workspace is not None + info_read, info_write = os.pipe() + block_read, block_write = os.pipe() + try: + os.set_inheritable(info_write, True) + os.set_inheritable(block_read, True) + group = await create_process_group_exec( + *verifier_workspace.bwrap_argv( + spawn, + cwd=workdir.as_posix(), + env={**verifier_env, **config.verifier.env}, + inherit_workspace_env=False, + info_fd=info_write, + userns_block_fd=block_read, + network=False, + mount_hosts=False, + isolate_processes=False, + ), + cwd=workdir, + env=environ, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + pass_fds=(info_write, block_read), + ) + os.close(info_write) + os.close(block_read) + info_write = block_read = -1 + await install_identity_map(info_read, block_write) + return group + finally: + os.close(info_read) + os.close(block_write) + for stray in (info_write, block_read): + if stray != -1: + os.close(stray) + + if workspace is None or severed or not workspace.owns_netns: + return await _grade_with_verifier(config, LOGS, answer, run_tests) + sandbox = await workspace.sandbox_pid() + if sandbox is None: + return await _grade_with_verifier(config, LOGS, answer, run_tests) + + async with workspace.visiting(config.allowed_hosts("verifier")) as visitor_env: + verifier_env.update(visitor_env) + spawn = workspace.enter_argv( + sandbox, + command, + env={**verifier_env, **config.verifier.env}, + identity=verifier_uid, + inherit_workspace_env=False, + preserve_credentials=True, + no_new_privs=False, + ) + return await _grade_with_verifier(config, LOGS, answer, run_tests) + + +async def _grade_with_verifier( + config: TaskConfig, + logs: Path, + answer: Any, + run_tests: Callable[[], Awaitable[ProcessGroup]], +) -> dict[str, Any]: + """Run the verifier and shape its reward into a HUD grade.""" + timeout = config.verifier.timeout_sec or DEFAULT_VERIFIER_TIMEOUT + _reset_dir(logs / "verifier") + _write_no_follow(logs / "agent_answer.txt", "" if answer is None else str(answer)) + group = await run_tests() + proc = group.process + assert proc.stdout is not None and proc.stderr is not None + reading = ( + asyncio.create_task(proc.stdout.read()), + asyncio.create_task(proc.stderr.read()), + ) + try: + try: + await asyncio.wait_for(proc.wait(), timeout=timeout) + timed_out = False + except TimeoutError: + timed_out = True + await group.terminate() + out_bytes, err_bytes = await asyncio.gather(*reading) + except BaseException: + for reader in reading: + reader.cancel() + await group.terminate() + raise + if timed_out: + return { + "score": 0.0, + "isError": True, + "content": f"Harbor verifier timed out after {timeout:.0f}s", + "info": { + "verifier_timeout_sec": timeout, + "stdout": out_bytes.decode("utf-8", "replace")[-4000:], + "stderr": err_bytes.decode("utf-8", "replace")[-4000:], + }, + } + + reward, info = _read_reward(logs / "verifier") + info.update( + { + "stdout": out_bytes.decode("utf-8", "replace")[-4000:], + "stderr": err_bytes.decode("utf-8", "replace")[-4000:], + } + ) + if reward is None: + return { + "score": 0.0, + "isError": True, + "content": "Harbor verifier did not write reward.json or reward.txt", + "info": info, + } + return {"score": reward, "info": info} + + +def _read_reward(verifier_logs: Path) -> tuple[float | None, dict[str, Any]]: + """Read a finite reward from reward.json or reward.txt.""" + reward_json = verifier_logs / "reward.json" + if reward_json.is_file(): + try: + data = json.loads(reward_json.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None, {"parse_error": "reward.json is not valid JSON"} + if (score := _as_score(data)) is not None: + return score, {"reward_file": str(reward_json)} + if isinstance(data, dict): + for key in ("reward", "score"): + if (score := _as_score(data.get(key))) is not None: + return score, {"reward_file": str(reward_json), "reward_json": data} + return None, {"reward_file": str(reward_json), "reward_parse_error": "no numeric reward"} + + reward_txt = verifier_logs / "reward.txt" + if reward_txt.is_file(): + text = reward_txt.read_text(encoding="utf-8").strip() + try: + value = float(text) + except ValueError: + return None, {"reward_file": str(reward_txt), "reward_parse_error": text} + if not math.isfinite(value): + return None, {"reward_file": str(reward_txt), "reward_parse_error": text} + return value, {"reward_file": str(reward_txt)} + + return None, {} + + +def _write_no_follow(path: Path, text: str) -> None: + """Write text without following a symlink at path.""" + if path.is_symlink(): + path.unlink() + with path.open("w", encoding="utf-8") as handle: + handle.write(text) + + +def _as_score(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) if math.isfinite(float(value)) else None diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index a1d174404..938863e9d 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -1,4 +1,4 @@ -"""The Harbor integration as data: load, provenance, grouping, adapt contexts. +"""The Harbor integration as data: load, provenance, adaptation, and runtime. Docker-side serving needs a daemon and is covered by the e2e integration scripts; here ``load``'s rows/provenance/stamping and ``adapt``'s build @@ -39,6 +39,14 @@ def _write_harbor_task(root: Path, name: str, marker: str = "FROM python:3.12-sl return task +@pytest.fixture(autouse=True) +def stub_docker(monkeypatch) -> None: + async def fake_docker(*args, **kwargs): + return b"", b"" + + monkeypatch.setattr("integrations.harbor._adapt._docker", fake_docker) + + def test_load_stamps_rows_with_provenance(tmp_path) -> None: _write_harbor_task(tmp_path, "task-a") _write_harbor_task(tmp_path, "task-b") @@ -54,9 +62,10 @@ async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None: _write_harbor_task(tmp_path, "task-a") _write_harbor_task(tmp_path, "task-b", marker="FROM python:3.11-slim\n") - images = await harbor.adapt(tmp_path, build=False) + taskset = await harbor.adapt(tmp_path) - assert images == {} + assert len(taskset) == 2 + assert all(task.runtime_config and task.runtime_config.image for task in taskset) contexts = sorted(p.name for p in (tmp_path / ".hud-adapt").iterdir()) assert len(contexts) == 2 # two env groups (distinct Dockerfiles) context = tmp_path / ".hud-adapt" / contexts[0] @@ -79,7 +88,7 @@ async def test_the_layer_keeps_its_own_state_out_of_the_task(tmp_path) -> None: # inside the task's own filesystem, where the agent finds a HUD runtime the # task never declared, and shims into a hidden tree resolving to nothing. _write_harbor_task(tmp_path, "task-a") - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) script = (context / "_hud" / "install.sh").read_text(encoding="utf-8") @@ -103,7 +112,7 @@ async def test_the_agent_toolchain_installs_only_what_the_image_lacks(tmp_path) # build time instead. An image shipping its own interpreter must keep it, # so every tool is presence-checked rather than installed outright. _write_harbor_task(tmp_path, "task-a") - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) script = (context / "_hud" / "install.sh").read_text(encoding="utf-8") decision = script[script.index('apt_pkgs=""') : script.index("\ndone") + len("\ndone")] @@ -137,18 +146,13 @@ def queued(path: str) -> set[str]: assert queued(str(stub)) == {"git", "curl", "ca-certificates"} -def test_adapt_images_stamp_rows_when_the_caller_passes_them(tmp_path) -> None: - # The mapping is a value the caller holds — nothing is written into the - # dataset, so there is no cache to go stale. +async def test_adapt_binds_built_images_to_rows(tmp_path) -> None: _write_harbor_task(tmp_path, "task-a") - ((env_name, _),) = harbor.grouped(tmp_path) - image = f"registry.io/acme/{env_name}:abc123" - - (task,) = list(harbor.load(tmp_path, images={env_name: image})) + (task,) = list(await harbor.adapt(tmp_path)) assert task.runtime_config is not None - assert task.runtime_config.image == image - # ...and without the mapping the rows carry only what the task declared. + assert task.runtime_config.image.startswith("hud-harbor-adapted:") + (bare,) = list(harbor.load(tmp_path)) assert bare.runtime_config is None or bare.runtime_config.image is None @@ -160,7 +164,7 @@ async def test_environment_serves_the_baked_tasks(tmp_path, monkeypatch) -> None monkeypatch.setattr("hud.environment.workspace.usable_bwrap", lambda: "/usr/bin/true") _write_harbor_task(tmp_path, "task-a") _write_harbor_task(tmp_path, "task-b") - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) env = harbor.environment(context / "_hud" / "tasks", name=context.name) @@ -179,6 +183,20 @@ def test_harbor_implements_the_integration_contract() -> None: assert harbor.integration.name == "harbor" +def test_public_surface_stays_at_the_integration_boundary() -> None: + assert set(harbor.__all__) == { + "adapt", + "detect", + "environment", + "export", + "integration", + "load", + } + assert not hasattr(harbor, "grouped") + assert not hasattr(harbor, "docker_runtime") + assert not hasattr(harbor, "agent_timeout") + + def test_load_translates_declared_requirements(tmp_path) -> None: task = _write_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text( @@ -199,7 +217,6 @@ def test_load_translates_declared_requirements(tmp_path) -> None: assert row.runtime_config.resources.gpu.count == 2 # Time budgets are the engine's, not the substrate's. assert row.runtime_config.limits is None - assert harbor_load.agent_timeout(task) == 2400.0 def test_load_omits_requirements_a_task_does_not_declare(tmp_path) -> None: @@ -238,7 +255,7 @@ async def test_served_templates_use_the_declared_description(tmp_path, monkeypat 'description = "Fix the thing properly."\n', encoding="utf-8", ) - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) env = harbor.environment(context / "_hud" / "tasks", name=context.name) @@ -256,10 +273,10 @@ async def test_multi_step_tasks_load_but_cannot_be_adapted_yet(tmp_path) -> None assert [row.id for row in harbor.load(tmp_path)] == ["task-a"] with pytest.raises(NotImplementedError, match="multi-step"): - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) -def test_declared_workspace_policy_is_translated(tmp_path) -> None: +def test_declared_workspace_contract_is_parsed_once(tmp_path) -> None: task = _write_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text( 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' @@ -268,24 +285,16 @@ def test_declared_workspace_policy_is_translated(tmp_path) -> None: encoding="utf-8", ) - policy = harbor_load.workspace_policy(task) - - assert policy == { - "network": True, - "env": {"TOKEN": "abc"}, - "agent_env": {}, - "workdir": "/srv/app", - # Public, but reached through the workspace's own way out rather than - # by sharing the substrate's network. - "allowed_hosts": ["*"], - # And the verifier's are its own, not a copy of the agent's. - "verifier_allowed_hosts": ["*"], - "user": None, - # Per phase, because Harbor's are: a task may restrict the agent and - # still verify as root. - "agent_user": None, - "verifier_user": None, - } + config = harbor_load.HarborTask.read(task).config + + assert config.network("agent") is True + assert config.environment.env == {"TOKEN": "abc"} + assert config.agent.env == {} + assert config.environment.workdir == "/srv/app" + assert config.allowed_hosts("agent") == frozenset({"*"}) + assert config.allowed_hosts("verifier") == frozenset({"*"}) + assert config.phase_user("agent") is None + assert config.phase_user("verifier") is None def test_each_phase_is_held_to_the_hosts_it_declared(tmp_path) -> None: @@ -298,10 +307,12 @@ def test_each_phase_is_held_to_the_hosts_it_declared(tmp_path) -> None: '[environment]\nnetwork_mode = "no-network"\n', encoding="utf-8", ) - assert harbor_load.unsupported_features(isolated) == [] - assert harbor_load.workspace_policy(isolated)["network"] is False - assert harbor_load.workspace_policy(isolated)["allowed_hosts"] == [] - assert harbor_load.workspace_policy(isolated)["verifier_allowed_hosts"] == [] + isolated_task = harbor_load.HarborTask.read(isolated) + isolated_config = isolated_task.config + assert isolated_task.unsupported_features() == [] + assert isolated_config.network("agent") is False + assert isolated_config.allowed_hosts("agent") == frozenset() + assert isolated_config.allowed_hosts("verifier") == frozenset() # Declared for the whole environment: both phases named those hosts. shared = _write_harbor_task(tmp_path, "shared") @@ -310,9 +321,11 @@ def test_each_phase_is_held_to_the_hosts_it_declared(tmp_path) -> None: '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', encoding="utf-8", ) - assert harbor_load.unsupported_features(shared) == [] - assert harbor_load.workspace_policy(shared)["allowed_hosts"] == ["pypi.org"] - assert harbor_load.workspace_policy(shared)["verifier_allowed_hosts"] == ["pypi.org"] + shared_task = harbor_load.HarborTask.read(shared) + shared_config = shared_task.config + assert shared_task.unsupported_features() == [] + assert shared_config.allowed_hosts("agent") == frozenset({"pypi.org"}) + assert shared_config.allowed_hosts("verifier") == frozenset({"pypi.org"}) # Declared for the agent alone: the verifier said nothing, so it is public. agent_only = _write_harbor_task(tmp_path, "agent-only") @@ -321,9 +334,11 @@ def test_each_phase_is_held_to_the_hosts_it_declared(tmp_path) -> None: '[agent]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', encoding="utf-8", ) - assert harbor_load.unsupported_features(agent_only) == [] - assert harbor_load.workspace_policy(agent_only)["allowed_hosts"] == ["pypi.org"] - assert harbor_load.workspace_policy(agent_only)["verifier_allowed_hosts"] == ["*"] + agent_task = harbor_load.HarborTask.read(agent_only) + agent_config = agent_task.config + assert agent_task.unsupported_features() == [] + assert agent_config.allowed_hosts("agent") == frozenset({"pypi.org"}) + assert agent_config.allowed_hosts("verifier") == frozenset({"*"}) def test_tasks_with_different_policies_get_separate_envs(tmp_path) -> None: @@ -347,7 +362,7 @@ def test_adapted_cmd_serves_the_contract_constructor(tmp_path) -> None: _write_harbor_task(tmp_path, "task-a") asyncio.get_event_loop_policy() - asyncio.run(harbor.adapt(tmp_path, build=False)) + asyncio.run(harbor.adapt(tmp_path)) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") @@ -360,7 +375,7 @@ async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> No import asyncio import json - from integrations.harbor._adapt import _grade_with_verifier + from integrations.harbor._runtime import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -393,7 +408,7 @@ async def test_a_declared_identity_governs_the_phases_not_the_harness(tmp_path) (task / "environment" / "Dockerfile").write_text( "FROM python:3.12-slim\nRUN useradd -m agent\nUSER agent\n", encoding="utf-8" ) - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) layer = (context / "Dockerfile").read_text(encoding="utf-8").split("adaptation layer", 1)[1] @@ -407,18 +422,18 @@ async def test_a_declared_identity_governs_the_phases_not_the_harness(tmp_path) def test_each_phase_takes_the_identity_declared_for_it(tmp_path, monkeypatch) -> None: # A task may hand the agent a restricted account and still verify as root. - from integrations.harbor import _adapt + from integrations.harbor import _runtime task = _write_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text( 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[agent]\nuser = "agent"\n', encoding="utf-8", ) - monkeypatch.setattr(_adapt, "_image_user", lambda _task: None) - monkeypatch.setattr(_adapt.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=1000)) + monkeypatch.setattr(_runtime, "_image_user", lambda: None) + monkeypatch.setattr(_runtime.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=1000)) - assert _adapt.phase_uid(task, "agent") == 1000 - assert _adapt.phase_uid(task, "verifier") is None # root, as declared by omission + assert _runtime.phase_uid(task, "agent") == 1000 + assert _runtime.phase_uid(task, "verifier") is None # root, as declared by omission def test_build_stage_entrypoint_does_not_refuse(tmp_path) -> None: @@ -431,7 +446,7 @@ def test_build_stage_entrypoint_does_not_refuse(tmp_path) -> None: encoding="utf-8", ) - assert harbor_load.unsupported_features(task) == [] + assert harbor_load.HarborTask.read(task).unsupported_features() == [] def test_declared_uid_zero_beats_the_source_user(tmp_path) -> None: @@ -444,13 +459,13 @@ def test_declared_uid_zero_beats_the_source_user(tmp_path) -> None: encoding="utf-8", ) - assert harbor_load.workspace_policy(task)["user"] == 0 + assert harbor_load.HarborTask.read(task).config.agent.user == 0 def test_rewards_are_finite_numbers_not_booleans(tmp_path) -> None: import json as jsonlib - from integrations.harbor._adapt import _read_reward + from integrations.harbor._runtime import _read_reward logs = tmp_path / "verifier" logs.mkdir() @@ -502,16 +517,16 @@ def test_an_invalid_task_toml_is_an_error_not_a_default(tmp_path) -> None: def test_grading_directories_do_not_exist_during_the_agent_phase(tmp_path, monkeypatch) -> None: """The image ships /tests and the verdict dir empty, and a previous rollout leaves them behind — either way the agent phase must not find them.""" - from integrations.harbor import _adapt + from integrations.harbor import _runtime tests, verdict = tmp_path / "tests", tmp_path / "logs" / "verifier" for stale in (tests, verdict): stale.mkdir(parents=True) (tests / "test.sh").write_text("the assertions", encoding="utf-8") - monkeypatch.setattr(_adapt, "TESTS", tests) - monkeypatch.setattr(_adapt, "VERIFIER_LOGS", verdict) + monkeypatch.setattr(_runtime, "TESTS", tests) + monkeypatch.setattr(_runtime, "VERIFIER_LOGS", verdict) - _adapt._hide_grading_dirs() + _runtime._hide_grading_dirs() assert not tests.exists() assert not verdict.exists() @@ -535,7 +550,7 @@ def record(*args, **kwargs): monkeypatch.setattr("hud.environment.env.Workspace", record) _write_harbor_task(tmp_path, "task-a") - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) harbor.environment(context / "_hud" / "tasks", name=context.name) @@ -564,7 +579,7 @@ async def test_dataset_symlinks_are_never_dereferenced(tmp_path) -> None: task = _write_harbor_task(tmp_path / "ds", "task-a") (task / "tests" / "leak.txt").symlink_to(secret) - await harbor.adapt(tmp_path / "ds", build=False) + await harbor.adapt(tmp_path / "ds") (context,) = sorted((tmp_path / "ds" / ".hud-adapt").iterdir()) # The link is copied as a link: its target was never read, so no host @@ -657,7 +672,7 @@ def test_declarations_this_integration_cannot_reproduce_are_refused( encoding="utf-8", ) - assert expected in " ".join(harbor_load.unsupported_features(task)) + assert expected in " ".join(harbor_load.HarborTask.read(task).unsupported_features()) async def test_a_refused_task_never_reaches_a_build_context(tmp_path) -> None: @@ -669,7 +684,7 @@ async def test_a_refused_task_never_reaches_a_build_context(tmp_path) -> None: ) with pytest.raises(NotImplementedError, match="healthcheck"): - await harbor.adapt(tmp_path, build=False) + await harbor.adapt(tmp_path) async def test_a_chatty_verifier_does_not_deadlock(tmp_path) -> None: @@ -678,7 +693,7 @@ async def test_a_chatty_verifier_does_not_deadlock(tmp_path) -> None: import asyncio from hud.utils.process import create_process_group_exec - from integrations.harbor._adapt import _grade_with_verifier + from integrations.harbor._runtime import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -714,13 +729,13 @@ def test_phase_env_reaches_the_phase_that_declared_it(tmp_path) -> None: encoding="utf-8", ) - policy = harbor_load.workspace_policy(task) - config = harbor_load.TaskConfig.read(task) + parsed = harbor_load.HarborTask.read(task) + config = parsed.config # Container-wide reaches every process; the agent's reaches its sessions # only; the verifier's is applied where the verifier runs. - assert policy["env"] == {"SHARED": "both"} - assert policy["agent_env"] == {"AGENT_ONLY": "yes"} + assert config.environment.env == {"SHARED": "both"} + assert config.agent.env == {"AGENT_ONLY": "yes"} assert config.verifier.env == {"VERIFIER_ONLY": "yes"} @@ -732,8 +747,9 @@ def test_only_a_real_user_conflict_is_refused(tmp_path) -> None: ) # One phase naming an identity is fine: the image's single USER is it. - assert harbor_load.unsupported_features(task) == [] - assert harbor_load.TaskConfig.read(task).user == "app" + parsed = harbor_load.HarborTask.read(task) + assert parsed.unsupported_features() == [] + assert parsed.config.agent.user == "app" def test_an_explicit_zero_timeout_is_not_silently_extended(tmp_path) -> None: @@ -753,7 +769,7 @@ async def test_a_cancelled_grade_leaves_nothing_running(tmp_path) -> None: import asyncio from hud.utils.process import create_process_group_exec - from integrations.harbor._adapt import _grade_with_verifier + from integrations.harbor._runtime import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -810,4 +826,4 @@ def test_a_workdir_inside_the_reserved_path_is_refused(tmp_path) -> None: f"FROM python:3.12-slim\nWORKDIR {harbor_load.HUD_ROOT}/app\n", encoding="utf-8" ) - assert "reserved" in " ".join(harbor_load.unsupported_features(task)) + assert "reserved" in " ".join(harbor_load.HarborTask.read(task).unsupported_features()) diff --git a/integrations/harbor/tests/test_harbor.py b/integrations/harbor/tests/test_harbor.py index 6547546e4..8a6af370d 100644 --- a/integrations/harbor/tests/test_harbor.py +++ b/integrations/harbor/tests/test_harbor.py @@ -226,6 +226,7 @@ async def test_scripts_drive_hud_task_lifecycle(tmp_path: Path) -> None: # Boot serves the channel, parks the run via setup, then hands off. assert "hud serve env:env" in boot assert "hud task start solve" in boot + assert "python3 -c" not in boot assert 'exec "$@"' in boot # Verifier grades the parked run and writes the Harbor reward. assert "hud task grade solve" in test_sh From 271cabeb2e1e748f917aa0a9ea3ef6f37edf57c3 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 01:28:36 -0700 Subject: [PATCH 22/33] feat(runtime): make Docker workspace isolation unconditional --- docs/v6/reference/runtime.mdx | 17 +++++----- hud/eval/docker-seccomp.json | 21 ++++++++++++ hud/eval/runtime.py | 27 +++++++-------- hud/eval/tests/test_docker_provider.py | 47 +++++++++++++++++++++----- pyproject.toml | 1 + 5 files changed, 82 insertions(+), 31 deletions(-) create mode 100644 hud/eval/docker-seccomp.json diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index cb378f708..287bf2925 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -115,21 +115,20 @@ SubprocessRuntime(path, *, env=None, ready_timeout=120.0) ### `DockerRuntime` ```python -DockerRuntime(image=None, *, port=8765, run_args=(), runtime_config=None, nested_sandbox=False) +DockerRuntime(image=None, *, port=8765, run_args=(), runtime_config=None) ``` - **`image`** - image name to run; shorthand for `runtime_config.image`. - **`port`** - port the image's CMD serves inside the container (the scaffolded `Dockerfile.hud` serves `8765`). - **`run_args`** - extra `docker run` flags, e.g. `["--gpus", "all"]` or `["-e", "KEY=VAL"]`. - **`runtime_config`** - a `RuntimeConfig` (image, resources) for finer control. -- **`nested_sandbox`** - the image sandboxes *inside* the container (a `Workspace` using bubblewrap). Off by default. - -An image that sandboxes internally needs nested user/mount/proc namespaces, -which Docker's default seccomp profile and masked `/proc` block. Passing -`nested_sandbox=True` starts the container with `--security-opt -seccomp=unconfined --security-opt systempaths=unconfined`; without it such an -environment refuses to serve rather than running unisolated. Images that do -not sandbox internally keep Docker's full containment, so this is opt-in. + +`DockerRuntime` always starts with HUD's compact seccomp profile and the system +path configuration required by `Workspace`'s bubblewrap sessions. The profile +is intentionally default-allow for Docker compatibility, allowing the +namespace syscalls needed for the inner workspace sandbox while denying a +small set of unrelated kernel interfaces. It is not Docker's default +deny-by-default profile. ### `ModalRuntime` diff --git a/hud/eval/docker-seccomp.json b/hud/eval/docker-seccomp.json new file mode 100644 index 000000000..3b1ee17e0 --- /dev/null +++ b/hud/eval/docker-seccomp.json @@ -0,0 +1,21 @@ +{ + "defaultAction": "SCMP_ACT_ALLOW", + "defaultErrnoRet": 1, + "syscalls": [ + { + "names": [ + "bpf", + "keyctl", + "kcmp", + "name_to_handle_at", + "open_by_handle_at", + "perf_event_open", + "process_vm_readv", + "process_vm_writev", + "ptrace", + "userfaultfd" + ], + "action": "SCMP_ACT_ERRNO" + } + ] +} diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index be37005a5..0be85303c 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -158,13 +158,16 @@ def _modal_image_from_uri(modal: Any, image_uri: str) -> Any: return modal.Image.from_registry(image_uri) -#: What a container needs to sandbox *inside itself*: bubblewrap's nested -#: user/mount/proc namespaces are blocked by Docker's default seccomp profile -#: and masked ``/proc``. Off by default — an env that does not sandbox -#: internally keeps full container isolation. -NESTED_SANDBOX_SECURITY_ARGS = ( +#: DockerRuntime always serves HUD environments, so this is part of the +#: provider contract rather than a per-image option. This is intentionally a +#: default-allow compatibility profile: Workspace's bwrap sessions need the +#: namespace and mount syscalls, while unrelated kernel interfaces stay denied. +_DOCKER_SECCOMP_PROFILE = Path(__file__).with_name("docker-seccomp.json") +_DOCKER_SECURITY_ARGS = ( "--security-opt", - "seccomp=unconfined", + f"seccomp={_DOCKER_SECCOMP_PROFILE}", + # Docker exposes system-path masking only as an all-or-nothing option; + # bwrap replaces the container's proc and dev mounts while building a wall. "--security-opt", "systempaths=unconfined", ) @@ -447,7 +450,9 @@ class DockerRuntime: container (the scaffolded ``Dockerfile.hud`` serves 8765). Each acquisition publishes that port on an ephemeral loopback port, yields its :class:`Runtime`, and force-removes the container on exit. *run_args* are - extra provider-specific ``docker run`` flags (``-e``, volumes). + extra provider-specific ``docker run`` flags (``-e``, volumes). Every + container gets HUD's nested-workspace security profile so its environment + can use bubblewrap-backed :class:`~hud.environment.Workspace` sessions. Acquisition returns as soon as the port mapping exists — the env may still be importing behind it. Protocol-level readiness is the client's @@ -461,15 +466,9 @@ def __init__( port: int = 8765, run_args: Sequence[str] = (), runtime_config: RuntimeConfig | dict[str, Any] | None = None, - nested_sandbox: bool = False, ) -> None: self.port = port self.run_args = tuple(run_args) - #: Whether the image sandboxes *inside* the container (a workspace - #: using bubblewrap). Relaxing seccomp and unmasking /proc is what - #: nested namespaces need, and what an image that never sandboxes - #: internally should not be given. - self.nested_sandbox = nested_sandbox config = RuntimeConfig(image=image) if image is not None else RuntimeConfig() if runtime_config is not None: config = config.with_overrides(RuntimeConfig.model_validate(runtime_config)) @@ -499,9 +498,9 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: out, _ = await _docker( "run", "--detach", - *(NESTED_SANDBOX_SECURITY_ARGS if self.nested_sandbox else ()), *self.run_args, *resource_args, + *_DOCKER_SECURITY_ARGS, "--publish", f"127.0.0.1::{self.port}", config.image, diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index b0dabf6c5..33d549d1e 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -10,6 +10,7 @@ import asyncio import hashlib +import json import logging import os import sys @@ -20,6 +21,7 @@ import pytest +import hud.eval.runtime as runtime_module from hud.eval.runtime import ( DaytonaRuntime, DockerRuntime, @@ -123,6 +125,10 @@ async def _docker_calls(docker_log: Path) -> list[str]: return (await asyncio.to_thread(docker_log.read_text)).splitlines() +def _docker_security_args() -> str: + return " ".join(runtime_module._DOCKER_SECURITY_ARGS) + + @dataclass(frozen=True) class _ModalImageRef: kind: str @@ -439,7 +445,9 @@ async def test_acquisition_publishes_ephemeral_port_and_removes_container( async with provider(_row()) as runtime: assert runtime.url == "tcp://127.0.0.1:43210" calls = await _docker_calls(docker_log) - assert calls[0] == "run --detach -e X=1 --publish 127.0.0.1::8765 img:tag" + assert calls[0] == ( + f"run --detach -e X=1 {_docker_security_args()} --publish 127.0.0.1::8765 img:tag" + ) assert calls[1] == "port cid-42 8765" assert (await _docker_calls(docker_log))[-1] == "rm --force cid-42" @@ -465,7 +473,8 @@ async def test_runtime_config_supplies_image_and_resources( calls = await _docker_calls(docker_log) assert calls[0] == ( - "run --detach --cpus 2 --memory 4096m --gpus 1 --publish 127.0.0.1::8765 img:firefox" + f"run --detach --cpus 2 --memory 4096m --gpus 1 {_docker_security_args()} " + "--publish 127.0.0.1::8765 img:firefox" ) @@ -488,7 +497,8 @@ async def test_task_runtime_config_overrides_default_image( ) assert (await _docker_calls(docker_log))[0] == ( - "run --detach --cpus 2 --memory 4096m --publish 127.0.0.1::8765 img:task" + f"run --detach --cpus 2 --memory 4096m {_docker_security_args()} " + "--publish 127.0.0.1::8765 img:task" ) @@ -980,16 +990,37 @@ async def test_container_that_dies_before_serving_fails_with_its_logs( assert calls[-1] == "rm --force cid-42" # cleanup still runs on failure -async def test_nested_sandbox_relaxes_only_when_asked( +def test_docker_profile_allows_workspace_namespace_syscalls() -> None: + profile = json.loads(runtime_module._DOCKER_SECCOMP_PROFILE.read_text()) + denied = {name for rule in profile["syscalls"] for name in rule["names"]} + + assert profile["defaultAction"] == "SCMP_ACT_ALLOW" + assert { + "mount", + "pivot_root", + "setns", + "umount", + "umount2", + "unshare", + }.isdisjoint(denied) + assert { + "bpf", + "keyctl", + "perf_event_open", + "ptrace", + "userfaultfd", + } <= denied + + +async def test_docker_runtime_always_prepares_for_workspace_isolation( tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Images that sandbox inside themselves need nested namespaces; images - that do not keep Docker's full containment.""" _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - async with DockerRuntime("img:tag", nested_sandbox=True)(_row()): + async with DockerRuntime("img:tag")(_row()): pass calls = await _docker_calls(docker_log) - assert "seccomp=unconfined" in calls[0] + assert f"seccomp={runtime_module._DOCKER_SECCOMP_PROFILE}" in calls[0] + assert "seccomp=unconfined" not in calls[0] assert "systempaths=unconfined" in calls[0] diff --git a/pyproject.toml b/pyproject.toml index b0e2abafe..2d8065733 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,7 @@ packages = ["hud"] # Ensure py.typed is included in the package [tool.hatch.build.targets.wheel.force-include] "hud/py.typed" = "hud/py.typed" +"hud/eval/docker-seccomp.json" = "hud/eval/docker-seccomp.json" [project.optional-dependencies] # AI providers (openai, anthropic, google-genai) are now core dependencies; this From e849cbcfda63ff39141bb14fafb55821cd6da532 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 02:54:30 -0700 Subject: [PATCH 23/33] refactor(integrations): simplify Harbor adaptation --- docs/v6/advanced/harbor-convert.mdx | 67 +- docs/v6/more/faq.mdx | 2 +- docs/v6/reference/cli.mdx | 4 +- hud/environment/__init__.py | 2 - hud/environment/integration.py | 54 -- hud/environment/workspace.py | 2 +- hud/graders/bash.py | 26 +- hud/utils/process.py | 23 + integrations/__init__.py | 12 +- integrations/harbor/Dockerfile | 14 + integrations/harbor/__init__.py | 68 +- integrations/harbor/_adapt.py | 236 ------ integrations/harbor/_export.py | 424 ---------- integrations/harbor/_load.py | 485 ----------- integrations/harbor/_runtime.py | 388 --------- integrations/harbor/adapt.py | 330 ++++++++ integrations/harbor/env.py | 307 +++++++ integrations/harbor/export.py | 238 ++++++ integrations/harbor/install.sh | 53 ++ integrations/harbor/tests/test_contract.py | 928 +++++---------------- integrations/harbor/tests/test_harbor.py | 171 +--- 21 files changed, 1229 insertions(+), 2605 deletions(-) delete mode 100644 hud/environment/integration.py create mode 100644 integrations/harbor/Dockerfile delete mode 100644 integrations/harbor/_adapt.py delete mode 100644 integrations/harbor/_export.py delete mode 100644 integrations/harbor/_load.py delete mode 100644 integrations/harbor/_runtime.py create mode 100644 integrations/harbor/adapt.py create mode 100644 integrations/harbor/env.py create mode 100644 integrations/harbor/export.py create mode 100644 integrations/harbor/install.sh diff --git a/docs/v6/advanced/harbor-convert.mdx b/docs/v6/advanced/harbor-convert.mdx index 6a9df8aae..c1349f9e1 100644 --- a/docs/v6/advanced/harbor-convert.mdx +++ b/docs/v6/advanced/harbor-convert.mdx @@ -4,52 +4,25 @@ description: "Load Harbor tasks into the HUD runtime or export HUD tasks to Harb icon: "ship" --- -Everything that authors tasks - HUD's own `env.py`, platform rows, **Harbor** -task dirs - is a *frontend* that loads into the same primitives (`Environment`, -`Task`, `Taskset`). Integrations are **loaders, not converters**: no codegen -roundtrip to run foreign tasks. Each one implements -[`hud.environment.Integration`](https://github.com/hud-evals/hud-python/blob/main/hud/environment/integration.py) -- `load(ref) -> Taskset` and `environment(ref) -> Environment` - and ships as -the repository's `integrations/`. - -Harbor's agent works *inside* its container, so its environment constructor is -only meaningful in there. `adapt()` packages it: one HUD-speaking image per -distinct build context, whose CMD serves `harbor:environment`. -The same rows then run on any container placement. +`adapt()` turns Harbor task directories into the same `Environment`, `Task`, and +`Taskset` primitives used by authored HUD environments. It builds the Harbor +environment first, then wraps that image with the checked-in +`integrations/harbor/Dockerfile`, `env.py`, and `install.sh`. The generated +directory contains task data, not generated source. ## Prerequisites - A Harbor task directory - each task has `task.toml` + `instruction.md`, and usually an `environment/` (with a `Dockerfile`) and `tests/`. -## Load Harbor tasks +## Run Harbor tasks -`load(path)` parses a Harbor task dir (or a dataset of them) into a `Taskset` -directly - one row per task dir (`id` = the dir name), sharing one declarative -`Environment` per distinct `environment/` build context: +Build the images and receive a runnable taskset: ```python from integrations import harbor from hud.eval import DockerRuntime -assert harbor.detect("./terminal-bench") -taskset = harbor.load("./terminal-bench") - -for task in taskset: - print(task.env, task.id, task.columns["difficulty"]) -``` - -Each row carries what the task declared: `[metadata]` as `columns` (difficulty, -category, tags - the platform's filter/leaderboard facets) and -`[environment]` cpu/memory/gpu as `runtime_config`. Time budgets stay off the -row, since they bound the *rollout* rather than the substrate; pass the desired -budget as `rollout_timeout` when running. - -## Run Harbor tasks - -Build the images once, then place the rows anywhere: - -```python taskset = await harbor.adapt("./terminal-bench") job = await taskset.run(agent, runtime=DockerRuntime()) ``` @@ -57,24 +30,22 @@ job = await taskset.run(agent, runtime=DockerRuntime()) `DockerRuntime` supplies the inner workspace sandbox needed by adapted images; the setting is automatic for all HUD Docker environments. -`adapt(path, push="registry.io/acme")` pushes instead and returns rows bound to -the pushed images. It also writes the generated contexts under `.hud-adapt/`, -which `hud deploy` can use directly. +`adapt(path, push="registry.io/acme")` pushes the wrapped images and returns +rows bound to those refs. The readable build inputs are left under +`.hud-adapt/`; `adapt()` owns the two-image build because the wrapper's base is +the Harbor environment it just built. -The image also carries what the task declared about its environment: -`[environment.env]`, `workdir` and `user` become `ENV`, `WORKDIR` and `USER` -directives, so Docker applies them to every process - the agent's shells, the -verifier, the serving process alike. +Each row carries `[metadata]` as `columns` and CPU, memory, and GPU requirements +as `runtime_config`. The image's `tasks.json` carries workdir, environment, +network, and phase-user declarations into the authored HUD environment. Tasks whose declared behaviour cannot be reproduced faithfully are refused rather than silently downgraded. Adaptation replaces the container's own boot -process with the serving command, so anything depending on that boot process is -refused: a Dockerfile `ENTRYPOINT`, `healthcheck` (nothing would start the -services it awaits), and `mcp_servers` (nothing would start the servers they -point at) - alongside compose environments, `network_mode = "allowlist"`, a -verifier with its own environment, prebuilt `docker_image` environments, -non-linux `os`, TPUs, multi-step `[[steps]]` tasks, and an agent and verifier -that must run as *different* users (one image, one `USER`). +process with the HUD server, so `healthcheck` and `mcp_servers` declarations are +refused when they depend on that process. Compose environments, separate +verifier environments, non-Linux tasks, TPUs, skills directories, and +multi-step tasks are also refused. Prebuilt images, allowlists, and distinct +agent/verifier users are supported. ## Export HUD tasks to Harbor diff --git a/docs/v6/more/faq.mdx b/docs/v6/more/faq.mdx index ce0374653..cf1022ca2 100644 --- a/docs/v6/more/faq.mdx +++ b/docs/v6/more/faq.mdx @@ -100,7 +100,7 @@ Evals are a complete use on their own - write tasks, run them across models, rea -Yes. The Harbor integration loads Harbor-format tasks straight into a `Taskset` (`integrations.harbor.load`), no conversion round-trip needed. And a whole benchmark can become one generative task definition. See [Harbor interop](/v6/advanced/harbor-convert). +Yes. `integrations.harbor.adapt()` builds Harbor-format tasks as a runnable HUD `Taskset`. And a whole benchmark can become one generative task definition. See [Harbor interop](/v6/advanced/harbor-convert). diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index 27605ecf7..f65affda4 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -132,8 +132,8 @@ hud sync tasks my-taskset # publish tasks as a named taskset hud sync env # sync environment metadata ``` -External benchmark formats (currently Harbor) load directly into the runtime -as `Taskset`s - no conversion step. See [Harbor interop](/v6/advanced/harbor-convert). +External benchmark formats (currently Harbor) can be adapted into runnable +`Taskset`s. See [Harbor interop](/v6/advanced/harbor-convert). ## Inspect diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index 356d26b62..3f2baaa56 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -24,7 +24,6 @@ from .egress import Peer from .env import Answer, Environment -from .integration import Integration from .workspace import DEFAULT_SYSTEM_MOUNTS, Mount, MountKind, Workspace @@ -93,7 +92,6 @@ def load_environment( "Answer", "Capability", "Environment", - "Integration", "Mount", "MountKind", "Peer", diff --git a/hud/environment/integration.py b/hud/environment/integration.py deleted file mode 100644 index 8f69c578b..000000000 --- a/hud/environment/integration.py +++ /dev/null @@ -1,54 +0,0 @@ -"""The integration contract: a foreign task format as a frontend to HUD. - -An integration translates a foreign benchmark format into HUD's *what* — -:class:`~hud.eval.Taskset` rows and :class:`Environment` s — and never owns -execution: placement stays a format-agnostic execution-time concern -(:mod:`hud.eval.runtime`). No codegen roundtrip to run foreign tasks. - -The contract is two verbs: load the format's data as rows, and construct -the live environment those rows join (by env name; each row's template id -dispatches within it). *Where* the constructor can execute -depends on the format: in-process formats (a dataset plus scoring code) run -it anywhere; formats whose environment is a container filesystem run it -inside an image built for that purpose — packaging the constructor into -such images is a format extra (e.g. ``harbor.adapt``), not part of the -contract. - -Implementations live outside core — this repository's ``integrations/``, or -any installable package; core knows only this interface, and imports none of -them. -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, ClassVar - -if TYPE_CHECKING: - from pathlib import Path - - from hud.eval import Taskset - - from .env import Environment - - -class Integration(ABC): - """One foreign format's frontend. See module docstring for the contract.""" - - #: The format's identifier, and the scheme its loaded tasksets are - #: origin-stamped with (``:``). - name: ClassVar[str] - - @abstractmethod - def load(self, ref: str | Path) -> Taskset: - """Foreign data as rows, origin-stamped ``:``.""" - - @abstractmethod - def environment(self, ref: str | Path, *, name: str | None = None) -> Environment: - """The live environment serving *ref*'s tasks. - - *name* selects among several env groups when the ref has more than - one. Rows join the env by name; each row's template id dispatches - within it. Freshness is the placement's concern: providers call the - constructor per acquisition. - """ diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 2176781de..d6440ddfc 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -773,7 +773,7 @@ def bwrap_argv( def enter_argv( self, pid: int, - command: str | None = None, + command: str | list[str] | None = None, *, env: Mapping[str, str] | None = None, identity: int | None | Literal["workspace"] = "workspace", diff --git a/hud/graders/bash.py b/hud/graders/bash.py index bb23df421..2ef60b94e 100644 --- a/hud/graders/bash.py +++ b/hud/graders/bash.py @@ -26,7 +26,7 @@ async def compute_score( cls, command: str, cwd: str | None = None, - timeout_seconds: int | None = None, + timeout_seconds: float | None = None, **kwargs: Any, ) -> SubScore: """Run ``command`` via ``bash -lc`` and score by exit code.""" @@ -37,7 +37,7 @@ async def compute_score( "Running grader command: %s (cwd=%s, timeout=%ss)", command, cwd, timeout_seconds ) try: - proc = await create_process_group_exec( + process = await create_process_group_exec( "/bin/bash", "-lc", command, @@ -45,34 +45,34 @@ async def compute_score( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout_bytes, stderr_bytes = await proc.communicate(max_wait=timeout_seconds) - stdout = stdout_bytes.decode(errors="replace") - stderr = stderr_bytes.decode(errors="replace") - returncode = proc.returncode if proc.returncode is not None else 1 - except TimeoutError: + except FileNotFoundError: return SubScore( name=cls.name, value=0.0, info={ "exit_code": None, "stdout": "", - "stderr": "", - "timed_out": True, - "timeout": timeout_seconds, + "stderr": "/bin/bash not found", + "timed_out": False, }, ) - except FileNotFoundError: + result = await process.complete(max_wait=timeout_seconds) + if result.timed_out: return SubScore( name=cls.name, value=0.0, info={ "exit_code": None, "stdout": "", - "stderr": "/bin/bash not found", - "timed_out": False, + "stderr": "", + "timed_out": True, + "timeout": timeout_seconds, }, ) + stdout = result.stdout.decode(errors="replace") + stderr = result.stderr.decode(errors="replace") + returncode = result.returncode if result.returncode is not None else 1 return SubScore( name=cls.name, value=1.0 if returncode == 0 else 0.0, diff --git a/hud/utils/process.py b/hud/utils/process.py index fcbd79f6f..9f800e845 100644 --- a/hud/utils/process.py +++ b/hud/utils/process.py @@ -12,6 +12,16 @@ _PROCESS_EXIT_POLL_INTERVAL = 0.05 +@dataclass(frozen=True, slots=True) +class ProcessResult: + """Captured outcome of a managed process group.""" + + returncode: int | None + stdout: bytes + stderr: bytes + timed_out: bool = False + + @dataclass(slots=True) class ProcessGroup: """Subprocess whose descendants share a teardown boundary. @@ -75,6 +85,19 @@ async def communicate( await self.terminate() return result + async def complete( + self, + input: bytes | None = None, + *, + max_wait: float | None = None, + ) -> ProcessResult: + """Capture output and teardown, reporting timeout as process data.""" + try: + stdout, stderr = await self.communicate(input, max_wait=max_wait) + except TimeoutError: + return ProcessResult(self.returncode, b"", b"", timed_out=True) + return ProcessResult(self.returncode, stdout, stderr) + async def terminate(self) -> None: await _terminate_process_group( self.process, diff --git a/integrations/__init__.py b/integrations/__init__.py index 6cbbfd176..597c7df51 100644 --- a/integrations/__init__.py +++ b/integrations/__init__.py @@ -1,11 +1 @@ -"""Interop frontends: foreign benchmark formats as HUD primitives. - -These live outside the ``hud`` package and ship no part of it — core knows -only :class:`hud.environment.Integration` and imports no implementation. - -Each format implements that contract (``load`` / ``environment``) and keeps -an ergonomic function surface (``harbor.load(...)``) plus format extras: -``harbor.detect`` recognizes the layout, ``harbor.adapt`` packages the -constructor into container images and returns bound rows, and -``harbor.export`` is the reverse direction. -""" +"""Repository-local adapters for external task formats.""" diff --git a/integrations/harbor/Dockerfile b/integrations/harbor/Dockerfile new file mode 100644 index 000000000..3118ca4f3 --- /dev/null +++ b/integrations/harbor/Dockerfile @@ -0,0 +1,14 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +ARG HUD_REQUIREMENT=hud +COPY env.py install.sh tasks.json /media/hud/ +COPY tasks /media/hud/tasks +COPY packages /media/hud/packages +RUN sh /media/hud/install.sh "${HUD_REQUIREMENT}" + +ENV HUD_SKIP_VERSION_CHECK=1 +EXPOSE 8765 +ENTRYPOINT [] +CMD ["/media/hud/venv/bin/hud", "serve", "/media/hud/env.py", "--host", "0.0.0.0", "--port", "8765"] diff --git a/integrations/harbor/__init__.py b/integrations/harbor/__init__.py index 629c4f03b..2bd21aedc 100644 --- a/integrations/harbor/__init__.py +++ b/integrations/harbor/__init__.py @@ -1,66 +1,10 @@ -"""Harbor (terminal-bench layout) interop: load, adapt, export. +"""Harbor task interop. -Harbor task structure:: - - task_name/ - ├── instruction.md # agent prompt - ├── task.toml # config: timeouts, metadata - ├── environment/Dockerfile # container the agent works in - ├── tests/test.sh # verification -> writes reward.txt / .json - └── solution/ # optional (ignored) - -Harbor's agent works *inside* its container, so :func:`environment` (the -:class:`~hud.environment.Integration` constructor) is meaningful only in -there. :func:`adapt` builds the images and returns the loaded rows with those -images bound to them:: - - taskset = await harbor.adapt("./tasks") - job = await taskset.run(agent, runtime=DockerRuntime()) - - taskset = await harbor.adapt("./tasks", push="registry.io/x") - job = await taskset.run(agent, runtime=HUDRuntime()) - -Plus :func:`export`, the reverse direction (HUD tasks -> Harbor folders). -Compose-based and prebuilt-``docker_image`` tasks are not supported yet. +``adapt()`` builds Harbor task directories as runnable HUD tasksets. +``export()`` writes HUD tasks back to Harbor directories. """ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from hud.environment import Integration - -if TYPE_CHECKING: - from pathlib import Path - - from hud.environment import Environment - from hud.eval import Taskset - -from ._adapt import adapt -from ._export import export -from ._load import detect, load -from ._runtime import environment - - -class _Harbor(Integration): - """The :class:`~hud.environment.Integration` contract for Harbor.""" - - name = "harbor" - - def load(self, ref: str | Path) -> Taskset: - return load(ref) - - def environment(self, ref: str | Path, *, name: str | None = None) -> Environment: - return environment(ref, name=name) - - -integration = _Harbor() +from .adapt import adapt +from .export import export -__all__ = [ - "adapt", - "detect", - "environment", - "export", - "integration", - "load", -] +__all__ = ["adapt", "export"] diff --git a/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py deleted file mode 100644 index c4b77d1c3..000000000 --- a/integrations/harbor/_adapt.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Build adapted Harbor images.""" - -from __future__ import annotations - -import json -import logging -import re -import shlex -import shutil -from pathlib import Path -from typing import TYPE_CHECKING - -from hud.utils.docker import docker as _docker - -from ._load import HUD_ROOT, HarborTask, _load, _task_groups, hash_directory - -if TYPE_CHECKING: - from hud.eval import Taskset - -LOGGER = logging.getLogger(__name__) - -_DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") -_DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*") -SERVING_PYTHON = "3.12" - -_CONTEXT_IGNORE_NAMES = ( - "__pycache__", - "*.pyc", - ".git", - ".venv", - "venv", - "*.egg-info", - ".pytest_cache", -) -_CONTEXT_IGNORE = shutil.ignore_patterns(*_CONTEXT_IGNORE_NAMES) - -_INSTALL_SH = """\ -#!/bin/sh -set -eu -export UV_INSTALL_DIR=__HUD_ROOT__/bin \ - UV_PYTHON_INSTALL_DIR=__HUD_ROOT__/python \ - UV_PYTHON_BIN_DIR=__HUD_ROOT__/bin \ - UV_NO_CACHE=1 \ - XDG_CONFIG_HOME=__HUD_ROOT__/config -export PATH="__HUD_ROOT__/bin:$PATH" -command -v uv >/dev/null 2>&1 || { - { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \ - || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \ - || apk add --no-cache curl ca-certificates - { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \ - || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \ - || pip install -q -U uv -} -command -v bwrap >/dev/null 2>&1 \ - || { apt-get update -qq && apt-get install -y -qq bubblewrap; } \ - || apk add --no-cache bubblewrap \ - || echo "warning: bubblewrap unavailable; tasks declaring isolation will refuse to serve" -uv python install __PYTHON__ -uv venv __HUD_ROOT__/venv --python __PYTHON__ -uv pip install --python __HUD_ROOT__/venv/bin/python __HUD_REQUIREMENT__ - -apt_pkgs="" -apk_pkgs="" -for spec in \ - "python3|python3 python3-venv|python3" \ - "pip3|python3-pip|py3-pip" \ - "git|git|git" \ - "curl|curl ca-certificates|curl ca-certificates" -do - if command -v "${spec%%|*}" >/dev/null 2>&1; then continue; fi - rest=${spec#*|} - apt_pkgs="$apt_pkgs ${rest%%|*}" - apk_pkgs="$apk_pkgs ${rest##*|}" -done -if [ -n "$apt_pkgs" ]; then - { apt-get update -qq && apt-get install -y -qq $apt_pkgs; } \ - || apk add --no-cache $apk_pkgs \ - || echo "warning: could not provision the agent toolchain:$apt_pkgs" -fi -""" - -_LAYER = """ - -# HUD adaptation layer -USER root -COPY _hud __HUD_ROOT__ -RUN sh __HUD_ROOT__/install.sh -COPY _hud_harbor __HUD_ROOT__/venv/lib/python__PYTHON__/site-packages/harbor -ENV HUD_SKIP_VERSION_CHECK=1 -EXPOSE 8765 -__DECLARED__ENTRYPOINT [] -CMD ["__HUD_ROOT__/venv/bin/hud", "serve", \ - "harbor:environment", "--arg", "ref=__HUD_ROOT__/tasks", \ - "--arg", "name=__ENV_NAME__", "--host", "0.0.0.0", "--port", "8765"] -""" - - -def _validated_user(declared: str | int | None, source_user: str | None) -> str | None: - """Validate a phase identity before it reaches a generated directive.""" - user = str(declared) if declared is not None else source_user - if user is None: - return None - if not all(_DOCKER_USER.fullmatch(part) for part in user.split(":", 1)): - raise ValueError(f"declared user {user!r} is not a usable user[:group]") - return user - - -def _declared_directives(task: HarborTask, source_user: str | None) -> str: - """Render the task's container-wide environment directives.""" - lines: list[str] = [] - for key, value in sorted(task.config.environment.env.items()): - if not _DOCKER_ENV_KEY.fullmatch(key): - raise ValueError(f"environment.env key {key!r} is not a usable variable name") - literal = json.dumps(value).replace("$", "\\$") - lines.append(f"ENV {key}={literal}") - if task.config.environment.workdir: - lines.append(f"WORKDIR {json.dumps(task.config.environment.workdir)}") - for role in ("agent", "verifier"): - _validated_user(task.config.phase_user(role), source_user) - return "".join(f"{line}\n" for line in lines) - - -async def adapt( - path: str | Path, - *, - push: str | None = None, - hud_requirement: str = "hud", -) -> Taskset: - """Build adapted images and return the runnable Harbor taskset.""" - root = Path(path).resolve() - out_root = root / ".hud-adapt" - images: dict[str, str] = {} - for env_name, group in _task_groups(root): - context = _write_context(out_root / env_name, env_name, group, hud_requirement) - content = hash_directory(context) - ref = f"{push}/{env_name}:{content}" if push else f"hud-harbor-adapted:{env_name}-{content}" - deadlines = [ - timeout for task in group if (timeout := task.config.environment.build_timeout_sec) - ] - await _docker( - "build", - "--tag", - ref, - str(context), - deadline=max(deadlines) if deadlines else None, - ) - if push: - await _docker("push", ref) - images[env_name] = ref - if images: - LOGGER.info("adapted %d image(s)", len(images)) - return _load(root, images=images) - - -def _write_context( - context: Path, env_name: str, group: list[HarborTask], hud_requirement: str -) -> Path: - """Write one group's environment and HUD adaptation layer.""" - if context.exists(): - shutil.rmtree(context) - source = group[0] - env_dir = source.path / "environment" - if not (env_dir / "Dockerfile").is_file(): - raise FileNotFoundError(f"group {env_name!r} has no environment/Dockerfile") - _copy_task_content(env_dir, context) - dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - - multi_step = [task.path.name for task in group if not (task.path / "instruction.md").is_file()] - if multi_step: - raise NotImplementedError( - "multi-step Harbor tasks (no root instruction.md) cannot be adapted yet: " - + ", ".join(sorted(multi_step)[:5]) - ) - for task in group: - if reasons := task.unsupported_features(): - raise NotImplementedError( - f"Harbor task {task.path.name!r} declares behaviour this integration " - f"cannot reproduce: {'; '.join(reasons)}" - ) - - dockerignore = context / ".dockerignore" - if dockerignore.is_file(): - _write( - dockerignore, - dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n!_hud_harbor\n!_hud_harbor/**\n", - ) - - hud_dir = context / "_hud" - hud_dir.mkdir(parents=True) - requirement = hud_requirement - wheel = Path(hud_requirement) - if wheel.suffix == ".whl" and wheel.is_file(): - shutil.copy2(wheel, hud_dir / wheel.name) - requirement = str(HUD_ROOT / wheel.name) - shutil.copytree( - Path(__file__).parent, - context / "_hud_harbor", - ignore=shutil.ignore_patterns("tests", ".hud-adapt", *_CONTEXT_IGNORE_NAMES), - ) - for task in group: - target = hud_dir / "tasks" / task.path.name - target.mkdir(parents=True) - for entry in ("instruction.md", "task.toml"): - _copy_task_content(task.path / entry, target / entry) - _copy_task_content(task.path / "tests", target / "tests") - _write( - hud_dir / "install.sh", - _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)) - .replace("__PYTHON__", SERVING_PYTHON) - .replace("__HUD_ROOT__", str(HUD_ROOT)), - ) - if source.final_stage.user is not None: - _write(hud_dir / "image-user", f"{source.final_stage.user}\n") - layer = ( - _LAYER.replace("__HUD_ROOT__", str(HUD_ROOT)) - .replace("__ENV_NAME__", env_name) - .replace("__PYTHON__", SERVING_PYTHON) - .replace( - "__DECLARED__", - _declared_directives(source, source.final_stage.user), - ) - ) - _write(context / "Dockerfile", dockerfile + layer) - return context - - -def _copy_task_content(source: Path, destination: Path) -> None: - """Copy task content without following symlinks.""" - if source.is_dir(): - shutil.copytree(source, destination, symlinks=True, ignore=_CONTEXT_IGNORE) - else: - shutil.copy2(source, destination, follow_symlinks=False) - - -def _write(path: Path, text: str) -> None: - path.write_text(text, encoding="utf-8", newline="\n") diff --git a/integrations/harbor/_export.py b/integrations/harbor/_export.py deleted file mode 100644 index 93cb97661..000000000 --- a/integrations/harbor/_export.py +++ /dev/null @@ -1,424 +0,0 @@ -"""Export HUD tasks to self-contained Harbor task folders. - -The reverse direction: a HUD task source becomes ``task.toml`` + -``instruction.md`` + ``environment/`` + ``tests/test.sh``. Convertible iff -the env's capabilities are ``ssh``/``mcp`` only (Harbor is shell-centric; -``rfb``/``cdp`` don't map). The exported image bakes an ENTRYPOINT that -serves the env control channel and runs the task's setup, parking the run so -``tests/test.sh`` can grade it over the channel — so it depends on that -ENTRYPOINT, not on a Harbor-native verifier. -""" - -from __future__ import annotations - -import json -import shlex -import shutil -from pathlib import Path -from typing import TYPE_CHECKING, Any - -from hud.environment import Environment -from hud.environment.server import TaskRunner -from hud.eval import Task, Taskset - -from ._load import dockerfile_instructions, final_stage, slugify - -if TYPE_CHECKING: - from collections.abc import Callable - -#: Capability protocols that map onto Harbor's shell/tool model. -ALLOWED_PROTOCOLS = ("ssh", "mcp") - -#: Where the agent writes its final answer (the contract between the -#: instruction and the verifier). Matches the Workspace default guest path. -DEFAULT_ANSWER_FILE = "/workspace/answer.txt" - -#: Port the in-container env control channel is served on. -CONTROL_PORT = 8765 - -#: Build-context entries never copied into the Harbor ``environment/`` dir. -_BUILD_CONTEXT_IGNORE = shutil.ignore_patterns( - "__pycache__", "*.pyc", ".git", ".venv", "venv", "*.egg-info", ".pytest_cache" -) - - -# ─── export: HUD tasks -> Harbor task folders ─────────────────────────── - - -def _write_text(path: Path, text: str) -> None: - """Write a generated file with LF endings (these run in Linux containers; - the default Windows ``\\r\\n`` translation breaks shebangs and shell scripts).""" - path.write_text(text, encoding="utf-8", newline="\n") - - -def _check_capabilities(env: Environment) -> None: - bad = [ - c.protocol for c in env.capabilities if c.protocol.split("/", 1)[0] not in ALLOWED_PROTOCOLS - ] - if bad: - raise ValueError( - f"env {env.name!r} declares non-Harbor capabilities {bad}; " - f"only {'/'.join(ALLOWED_PROTOCOLS)} are convertible.", - ) - - -def _safe_component(slug: str) -> str: - """A slug reduced to one path component that stays inside the output dir. - - Separators become hyphens so a namespaced id ("suite/fix") stays one - folder that ``harbor.load()`` can see. A slug with nothing nameable in it - ("..") is refused rather than silently renamed to a default. - """ - if not any(character.isalnum() for character in slug): - raise ValueError(f"task slug {slug!r} does not form a usable directory name") - return slugify(slug.replace("/", "-").replace("\\", "-")) - - -async def _materialize_prompt(env: Environment, task: str, args: dict[str, Any]) -> str: - """Run a task's first yield locally to get its concrete prompt (deterministic). - - The environment is started first: serving runs ``@env.initialize`` before - any task, so a template reading hook-published state would otherwise see - an uninitialized environment here and bake a different prompt than the - exported container produces. - """ - runner = TaskRunner(env.tasks[task], args) - try: - payload = await runner.start() - finally: - await runner.cancel() - prompt = payload.get("prompt") - return prompt if isinstance(prompt, str) else json.dumps(prompt, indent=2, default=str) - - -def _resolve_env(task: Task, authored: dict[str, Environment]) -> Environment: - """Resolve a task row's env name to a local, authored env defining the task. - - Rows reference envs by name; export materializes prompts in-process, so - the authored ``Environment`` must be defined in (or next to) the task - source. A row whose name matches nothing exportable fails loudly. - """ - env = authored.get(task.env) - if env is None or task.id not in env.tasks: - raise TypeError( - f"harbor export needs a local env defining task {task.id!r} " - f"(an env.py named {task.env!r} next to the tasks); none was found.", - ) - return env - - -# ─── generated files ─────────────────────────────────────────────────── - -_ENTRYPOINT_SH = """\ -#!/bin/sh -# Baked ENTRYPOINT (POSIX sh — slim base images have no bash): serve the HUD -# control channel, run the task setup (parking the paused run), then exec the -# container command. Harbor overrides CMD with `sleep infinity`, so setup must -# run via ENTRYPOINT; `exec "$@"` keeps the channel alive alongside it. The -# agent and the verifier both run in this same container, so the verifier -# reaches the parked run on 127.0.0.1:{port} to grade. -set -u - -hud serve {serve_target} --port {port} & - -# The SDK connection performs protocol-level readiness retries before setup. -hud task start {task} --args {args_json} --url tcp://127.0.0.1:{port} || {{ - echo "hud: task setup failed; refusing to run the agent against an unset task" >&2 - exit 1 -}} - -exec "$@" -""" - -_TEST_SH = """\ -#!/bin/sh -# Grade the parked HUD run against the agent's work, writing the Harbor reward. -set -u -mkdir -p /logs/verifier - -ANSWER_FILE={answer_file} -[ -f "$ANSWER_FILE" ] || : > "$ANSWER_FILE" - -# A grader that cannot run is not a score of 0: exiting nonzero lets Harbor -# record the trial as errored instead of as an agent that failed the task. -if ! hud task grade {task} --args {args_json} --answer-file "$ANSWER_FILE" \\ - --url tcp://127.0.0.1:{port} > /logs/verifier/reward.txt 2> /logs/verifier/grade.err; then - rm -f /logs/verifier/reward.txt - echo "hud: grading failed; see /logs/verifier/grade.err" >&2 - exit 1 -fi -""" - -_INSTRUCTION_SUFFIX = """\ - ---- -When you have finished, write your final answer to `{answer_file}`. -""" - - -def _adapt_env_dockerfile(content: str) -> str: - """Neutralize the env's own CMD/ENTRYPOINT and bake the boot ENTRYPOINT. - - ENTRYPOINT (not CMD) because Harbor overrides the container command with - ``sleep infinity``; our entrypoint runs setup then ``exec "$@"`` into it. - """ - lines = content.splitlines() - # A CMD/ENTRYPOINT may span backslash-continued lines; commenting only the - # first would leave the rest as invalid top-level instructions. - neutralized = { - number - for word, _, numbers in dockerfile_instructions(content) - if word in ("CMD", "ENTRYPOINT") - for number in numbers - } - lines = [ - f"# [hud original] {line}" if index in neutralized else line - for index, line in enumerate(lines) - ] - # COPY writes a root-owned file, so chmod needs root — and the image's - # own runtime identity is restored afterwards. - source_user = final_stage(content).user - boot_layer = ( - "\n# ─── HUD → Harbor boot entrypoint ───\n" - "USER root\n" - "COPY hud_entrypoint.sh /hud_entrypoint.sh\n" - "RUN chmod +x /hud_entrypoint.sh\n" - + (f"USER {source_user}\n" if source_user else "") - + 'ENTRYPOINT ["/hud_entrypoint.sh"]\n' - "# Default command for standalone `docker run`; Harbor injects its own.\n" - 'CMD ["sh", "-c", "sleep infinity"]\n' - ) - return "\n".join(lines) + "\n" + boot_layer - - -def _harbor_task_toml(name: str, task: str, args: dict[str, Any], timeout: float) -> str: - """A Harbor-native ``task.toml`` (``name``/``version`` required by the registry).""" - return ( - 'version = "1.0"\n' - f"name = {json.dumps(name)}\n" - "\n[metadata]\n" - f"hud_task = {json.dumps(task)}\n" - f"hud_args = {json.dumps(json.dumps(args))}\n" - "\n[agent]\n" - f"timeout_sec = {timeout}\n" - "\n[verifier]\n" - f"timeout_sec = {timeout}\n" - ) - - -def _find_dockerfile(source_dir: Path) -> Path | None: - return next( - (source_dir / n for n in ("Dockerfile.hud", "Dockerfile") if (source_dir / n).exists()), - None, - ) - - -def _make_ignore(out_root: Path) -> Callable[[str, list[str]], set[str]]: - """Ignore the standard caches plus the export output dir (which may live under - the source dir, e.g. ``./harbor_tasks`` next to ``env.py``).""" - out_resolved = out_root.resolve() - - def _ignore(dirpath: str, names: list[str]) -> set[str]: - ignored = set(_BUILD_CONTEXT_IGNORE(dirpath, names)) - base = Path(dirpath) - ignored.update(n for n in names if (base / n).resolve() == out_resolved) - return ignored - - return _ignore - - -def _write_environment( - task_dir: Path, - source_dir: Path, - dockerfile: Path, - taskset_file: Path | None, - serve_target: str, - task: str, - args: dict[str, Any], - out_root: Path, -) -> None: - """Copy the env build context into ``environment/`` and bake the boot entrypoint.""" - env_out = task_dir / "environment" - if env_out.exists(): - shutil.rmtree(env_out) - shutil.copytree(source_dir, env_out, ignore=_make_ignore(out_root), symlinks=True) - - # Drop the copied taskset file itself — and only that: a build context may - # legitimately need its own JSON (package.json, tsconfig.json), and a - # ``.py`` source stays because the exported image serves it. - if taskset_file is not None: - copied = env_out / taskset_file.name - if copied.is_file(): - copied.unlink() - dockerignore = env_out / ".dockerignore" - if dockerignore.is_file(): - # The task's ignore rules were written for its own build; they must - # not exclude the entrypoint this export generates (re-including a - # path needs its own rule — a directory rule does not cover it). - dockerignore.write_text( - dockerignore.read_text("utf-8") + "\n!hud_entrypoint.sh\n", - encoding="utf-8", - newline="\n", - ) - - for name in ("Dockerfile.hud", "dockerfile"): - leftover = env_out / name - if leftover.exists() and leftover.name != "Dockerfile": - leftover.unlink() - - _write_text(env_out / "Dockerfile", _adapt_env_dockerfile(dockerfile.read_text("utf-8"))) - _write_text( - env_out / "hud_entrypoint.sh", - _ENTRYPOINT_SH.format( - port=CONTROL_PORT, - serve_target=shlex.quote(serve_target), - task=shlex.quote(task), - args_json=shlex.quote(json.dumps(args)), - ), - ) - - -async def export( - source: str, - out_dir: str | Path, - *, - answer_file: str = DEFAULT_ANSWER_FILE, - timeout_sec: float = 600.0, -) -> list[Path]: - """Export HUD tasks from *source* into Harbor task folders under *out_dir*. - - The task's setup runs twice: once here, to capture ``instruction.md``, - and again inside the exported container at boot, where the run that gets - graded is parked. A task whose setup is not deterministic for its args - (randomized challenges, per-run state) will therefore grade a different - run than the one whose prompt was captured — such tasks need their - randomness moved into ``args`` before export. - - *source* is either a **tasks file** (``.json`` / ``.jsonl`` of ``{env, task, - args}`` entries) or a ``.py`` file/dir exposing ``Task``s. One folder is - written per task (task + args), each a self-contained Harbor task. Requires the - env's build context (a ``Dockerfile.hud``/``Dockerfile`` next to the source). - Returns the created task directories. - """ - from hud.utils.modules import iter_modules - - out = Path(out_dir).resolve() - out.mkdir(parents=True, exist_ok=True) - src = Path(source).resolve() - source_dir = src.parent if src.is_file() else src - - tasks = list(Taskset.from_file(src)) - # Rows reference envs by name; collect the authored envs (defined in the - # source, or next to a tasks file) to materialize prompts in-process. - scan = source_dir if src.suffix in (".json", ".jsonl") else src - authored: dict[str, Environment] = {} - # Remember the module attribute each env was found under: the exported - # container serves *that* target, not a guessed ``env:env``. - serve_targets: dict[str, str] = {} - for module in iter_modules(scan): - module_file = Path(getattr(module, "__file__", "") or "") - for attribute, value in vars(module).items(): - if isinstance(value, Environment): - authored[value.name] = value - serve_targets[value.name] = f"{module_file.stem or 'env'}:{attribute}" - - dockerfile = _find_dockerfile(source_dir) - if dockerfile is None: - raise FileNotFoundError( - f"no Dockerfile(.hud) next to {source_dir}; harbor export needs the env's " - "build context to rebuild the image under Harbor.", - ) - - created: list[Path] = [] - started: set[str] = set() - try: - created = await _export_tasks( - tasks, - authored, - serve_targets, - started, - out, - source_dir, - src, - dockerfile, - answer_file, - timeout_sec, - ) - finally: - for name in started: - await authored[name].stop() - return created - - -async def _export_tasks( - tasks: list[Task], - authored: dict[str, Environment], - serve_targets: dict[str, str], - started: set[str], - out: Path, - source_dir: Path, - src: Path, - dockerfile: Path, - answer_file: str, - timeout_sec: float, -) -> list[Path]: - created: list[Path] = [] - claimed: dict[str, str] = {} - for task in tasks: - env = _resolve_env(task, authored) - if env.name not in started: - # Serving runs @env.initialize before any task; a template reading - # hook-published state must see the same environment here. Recorded - # for teardown *before* starting: a hook that raises after an - # earlier one allocated would otherwise leak it. - started.add(env.name) - await env.start() - _check_capabilities(env) - - # A slug is user data that becomes a directory name: namespaced ids - # ("suite/fix") would nest out of harbor.load()'s reach and ".." would - # escape the output directory entirely. - declared = task.slug - slug = _safe_component(declared) - if slug in claimed: - raise ValueError( - f"task slugs {claimed[slug]!r} and {declared!r} both name the export " - f"directory {slug!r}; give them distinct slugs" - ) - claimed[slug] = declared - task_dir = out / slug - (task_dir / "tests").mkdir(parents=True, exist_ok=True) - - prompt = await _materialize_prompt(env, task.id, task.args) - instruction = prompt + _INSTRUCTION_SUFFIX.format(answer_file=answer_file) - _write_text(task_dir / "instruction.md", instruction) - - _write_text( - task_dir / "task.toml", - _harbor_task_toml(slug, task.id, task.args, timeout_sec), - ) - - _write_environment( - task_dir, - source_dir, - dockerfile, - src if src.is_file() and src.suffix in (".json", ".jsonl") else None, - serve_targets.get(env.name, "env:env"), - task.id, - task.args, - out, - ) - - _write_text( - task_dir / "tests" / "test.sh", - _TEST_SH.format( - port=CONTROL_PORT, - task=shlex.quote(task.id), - args_json=shlex.quote(json.dumps(task.args)), - answer_file=shlex.quote(answer_file), - ), - ) - - created.append(task_dir) - - return created diff --git a/integrations/harbor/_load.py b/integrations/harbor/_load.py deleted file mode 100644 index 465a65a9a..000000000 --- a/integrations/harbor/_load.py +++ /dev/null @@ -1,485 +0,0 @@ -"""Harbor task-dir parsing and loading: dirs -> Taskset rows with provenance.""" - -from __future__ import annotations - -import hashlib -import json -import logging -import os -import re -import tomllib -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field, ValidationError - -from hud.environment.egress import ANY_HOST -from hud.eval import Task, Taskset -from hud.eval.runtime import RuntimeConfig, RuntimeGPU, RuntimeResources -from hud.utils.naming import normalize_environment_name - -LOGGER = logging.getLogger(__name__) - -#: Where an adapted image keeps the harness: its venv, the baked tasks, the -#: session keys, the grading material. Nested rather than at the root, and -#: rebuilt out of the agent's namespace by :func:`~harbor.environment`, -#: so what the graded party sees is the empty ``/media`` any container has — -#: not a directory named after the thing evaluating it. -HUD_ROOT = Path("/media/hud") - -DEFAULT_VERIFIER_TIMEOUT = 600.0 - - -class _Phase(BaseModel): - """A phase Harbor runs: ``[agent]`` or ``[verifier]``.""" - - model_config = ConfigDict(extra="allow") - - timeout_sec: float | None = Field(default=None, gt=0) - user: str | int | None = None - network_mode: str | None = None - allowed_hosts: list[str] = Field(default_factory=list) - env: dict[str, str] = Field(default_factory=dict) - environment: dict[str, Any] | None = None - environment_mode: str | None = None - - -class _EnvironmentSection(BaseModel): - """``[environment]``: how the task's container is built and run.""" - - model_config = ConfigDict(extra="allow") - - build_timeout_sec: float | None = Field(default=None, gt=0) - docker_image: str | None = None - os: str | None = None - cpus: float | None = Field(default=None, ge=0) - memory_mb: int | None = Field(default=None, ge=0) - gpus: int | None = Field(default=None, ge=0) - gpu_types: list[str] = Field(default_factory=list) - tpu: dict[str, Any] | None = None - network_mode: str | None = None - allowed_hosts: list[str] = Field(default_factory=list) - workdir: str | None = None - env: dict[str, str] = Field(default_factory=dict) - healthcheck: dict[str, Any] | None = None - mcp_servers: list[dict[str, Any]] = Field(default_factory=list) - - -class _Package(BaseModel): - """``[task]``: what the task calls itself.""" - - model_config = ConfigDict(extra="allow") - - name: str | None = None - description: str | None = None - keywords: list[str] = Field(default_factory=list) - - -class TaskConfig(BaseModel): - """A task's ``task.toml``, as much of it as this integration consumes. - - The schema carries the coercion — a positive-number constraint instead of - a hand-rolled check, a typed table instead of ``isinstance(x, dict)`` at - every read — so the rest of the integration reads attributes. Unknown - keys are kept rather than rejected: Harbor's format grows, and a field - this integration does not consume is not an error. - """ - - model_config = ConfigDict(extra="allow") - - schema_version: str | None = None - task: _Package = Field(default_factory=_Package) - metadata: dict[str, Any] = Field(default_factory=dict) - environment: _EnvironmentSection = Field(default_factory=_EnvironmentSection) - agent: _Phase = Field(default_factory=_Phase) - verifier: _Phase = Field(default_factory=_Phase) - steps: list[dict[str, Any]] | None = None - - @classmethod - def read(cls, task_dir: Path) -> TaskConfig: - """Parse *task_dir*'s ``task.toml``. - - A file that will not parse declares nothing — its defaults apply. A - file that parses but declares something invalid (a GPU count that is - not a number, a timeout of zero) is an error: running it would grade - a task under requirements its author did not write. - """ - try: - raw = tomllib.loads((task_dir / "task.toml").read_text("utf-8")) - except (OSError, tomllib.TOMLDecodeError): - return cls() - try: - return cls.model_validate(raw) - except ValidationError as error: - raise ValueError( - f"{task_dir.name}/task.toml is not a valid Harbor task: {error}" - ) from error - - def phase(self, role: str) -> _Phase: - if role == "agent": - return self.agent - if role == "verifier": - return self.verifier - raise ValueError(f"unknown Harbor phase {role!r}") - - def network(self, role: str) -> bool: - """Whether *role*'s processes may reach the network at all. - - Harbor declares isolation container-wide or per phase; either severs - that phase, so the workspace and the verifier cannot disagree about - what one task declared. - """ - return "no-network" not in (self.environment.network_mode, self.phase(role).network_mode) - - def allowed_hosts(self, role: str) -> frozenset[str]: - """Which hosts *role* may reach. - - A phase declaring ``allowlist`` may reach the hosts it names and - nothing else — including nothing else on the substrate, which is where - the harness serving it lives. ``public`` names no hosts and so permits - all of them; ``no-network`` is the empty set, which permits none. - Every phase gets an answer: a task that says nothing about its network - is still held to something, rather than sharing the substrate's. - """ - mode = self.phase(role).network_mode or self.environment.network_mode or "public" - if mode == "no-network": - return frozenset() - if mode != "allowlist": - # Public still means every host, but reached the same way as any - # other policy: through the workspace's own way out. Sharing the - # substrate's network would make "public" mean the substrate's - # services too — the channel that grades the rollout among them. - return frozenset({ANY_HOST}) - declared = [*self.environment.allowed_hosts, *self.phase(role).allowed_hosts] - return frozenset(declared) - - def phase_user(self, role: str) -> str | int | None: - """The identity *role* runs as, if the task names one for it. - - Per phase, because Harbor's are: a task may hand the agent a - restricted account and still verify as root. They collapsed into one - value while the identity was a ``USER`` directive, which an image has - only one of; applied per phase, each keeps what it declared. - """ - return self.phase(role).user - - -def detect(path: str | Path) -> bool: - """True when *path* is a Harbor task dir or a dataset of them.""" - return bool(task_dirs(path)) - - -def load(path: str | Path) -> Taskset: - """Load a Harbor task dir (or dataset dir) into a :class:`Taskset`. - - One row per task dir (``id`` = the dir name); rows share one env name per - distinct environment. Each row carries the task's declared launch - requirements (cpu/memory/gpu). Adapted images are bound by :func:`adapt` - before the returned taskset is run:: - - taskset = await harbor.adapt(path) - job = await taskset.run(agent, runtime=DockerRuntime()) - """ - return _load(path) - - -def _load(path: str | Path, *, images: dict[str, str] | None = None) -> Taskset: - """Load rows, optionally binding image refs produced by :func:`adapt`.""" - root = Path(path).resolve() - dataset_name = root.parent.name if is_harbor_task(root) else root.name - - tasks: list[Task] = [] - for env_name, group in _task_groups(root): - image = (images or {}).get(env_name) - tasks.extend( - Task( - env=env_name, - id=task.path.name, - columns=task.columns, - runtime_config=task.runtime_config(image=image), - ) - for task in group - ) - return Taskset(slugify(dataset_name), tasks, origin=f"harbor:{root}") - - -@dataclass(frozen=True) -class FinalStage: - """What a Dockerfile's last stage declares. - - Adaptation appends to the final stage, so only that stage's state is - meaningful: a build stage's ``ENTRYPOINT`` starts nothing in the shipped - image, and its ``USER`` is not the shipped identity. Each ``FROM`` opens - a new stage whose inherited state comes from its base image — unknowable - from the text, so it reads as unset. - """ - - directives: frozenset[str] = frozenset() - user: str | None = None - workdir: str | None = None - - -def dockerfile_instructions(dockerfile_text: str) -> list[tuple[str, str, list[int]]]: - """Logical Dockerfile instructions: ``(word, operand, line numbers)``. - - A backslash continues an instruction onto the next line, so a physical - line is not an instruction — treating it as one both misreads operands - and, when rewriting, leaves continuation lines behind as invalid - top-level directives. A heredoc body is likewise not instructions: a - ``RUN <[\"']?[A-Za-z_][A-Za-z0-9_]*[\"']?)", joined) - ] - pending, numbers = [], [] - if heredocs: - numbers = [] - if pending: # trailing backslash at EOF - joined = " ".join(part for part in pending if part) - word, _, rest = joined.partition(" ") - instructions.append((word.upper(), rest.strip(), numbers)) - return instructions - - -def final_stage(dockerfile_text: str) -> FinalStage: - """Parse *dockerfile_text* into its :class:`FinalStage`.""" - directives: set[str] = set() - user: str | None = None - workdir: str | None = None - for word, operand, _ in dockerfile_instructions(dockerfile_text): - if word == "FROM": - directives, user, workdir = set(), None, None - elif word == "USER": - user = operand or None - elif word == "WORKDIR": - workdir = operand.strip().strip('"') or None - directives.add(word) - return FinalStage( - frozenset(directives), - None if user in ("root", "0", "root:root", "0:0") else user, - workdir, - ) - - -@dataclass(frozen=True, slots=True) -class HarborTask: - """The parsed, immutable view of one Harbor task directory. - - Loading, grouping, image adaptation, and runtime setup all consume this - same record. That keeps task.toml and Dockerfile parsing at one boundary - instead of making each phase rediscover the task from its path. - """ - - path: Path - config: TaskConfig - dockerfile: str - final_stage: FinalStage - environment_hash: str - - @classmethod - def read(cls, task_dir: Path) -> HarborTask: - path = Path(task_dir).resolve() - dockerfile_path = path / "environment" / "Dockerfile" - dockerfile = ( - dockerfile_path.read_text("utf-8", errors="replace") - if dockerfile_path.is_file() - else "" - ) - environment = path / "environment" - return cls( - path=path, - config=TaskConfig.read(path), - dockerfile=dockerfile, - final_stage=final_stage(dockerfile), - environment_hash=hash_directory(environment) if environment.exists() else "no-env", - ) - - @property - def columns(self) -> dict[str, Any] | None: - fields = dict(self.config.metadata) - if self.config.task.keywords: - fields.setdefault("keywords", self.config.task.keywords) - return fields or None - - def runtime_config(self, *, image: str | None = None) -> RuntimeConfig | None: - """Map portable launch requirements into the SDK's runtime config.""" - environment = self.config.environment - resources = RuntimeResources( - cpu=environment.cpus or None, - memory_mb=environment.memory_mb or None, - gpu=RuntimeGPU( - count=environment.gpus, - type=next((gpu_type for gpu_type in environment.gpu_types if gpu_type), None), - ) - if environment.gpus - else None, - ) - declared = RuntimeConfig( - image=image, - resources=resources if resources.model_dump(exclude_none=True) else None, - ) - return declared if declared.model_dump(exclude_none=True) else None - - @property - def workspace_key(self) -> str: - """Stable serialization of the workspace contract used for grouping.""" - config = self.config - return json.dumps( - { - "network": config.network("agent"), - "env": dict(config.environment.env), - "agent_env": dict(config.agent.env), - "workdir": config.environment.workdir or None, - "user": ( - config.agent.user if config.agent.user is not None else config.verifier.user - ), - "allowed_hosts": sorted(config.allowed_hosts("agent")), - "verifier_allowed_hosts": sorted(config.allowed_hosts("verifier")), - "agent_user": config.phase_user("agent"), - "verifier_user": config.phase_user("verifier"), - }, - sort_keys=True, - ) - - def unsupported_features(self) -> list[str]: - """Declarations this integration cannot reproduce faithfully.""" - config = self.config - environment, agent, verifier = config.environment, config.agent, config.verifier - reasons: list[str] = [] - - if environment.os not in (None, "linux"): - reasons.append(f"environment.os={environment.os!r}") - if environment.tpu: - reasons.append("environment.tpu (no TPU resource model)") - if agent.user is not None and verifier.user is not None and agent.user != verifier.user: - reasons.append("agent.user and verifier.user differ (the image has one USER)") - workdir = environment.workdir or self.final_stage.workdir - if workdir and Path(workdir).is_relative_to(HUD_ROOT): - reasons.append(f"working directory {workdir!r} is inside {HUD_ROOT} (reserved)") - if environment.docker_image and not (self.path / "environment" / "Dockerfile").is_file(): - reasons.append( - "prebuilt docker_image environments (adapt builds from environment/Dockerfile)" - ) - if "ENTRYPOINT" in self.final_stage.directives: - reasons.append( - "environment/Dockerfile ENTRYPOINT (adaptation replaces container startup)" - ) - if any( - (self.path / "environment" / name).is_file() - for name in ("docker-compose.yaml", "docker-compose.yml", "compose.yaml", "compose.yml") - ): - reasons.append("compose environments (sidecar services would never start)") - if environment.healthcheck: - reasons.append("environment.healthcheck (nothing starts the services it would await)") - if environment.mcp_servers: - reasons.append("environment.mcp_servers (nothing starts the servers they point at)") - if verifier.environment_mode or verifier.environment: - reasons.append("verifier runs in its own environment") - if config.steps: - reasons.append("multi-step tasks ([[steps]])") - return reasons - - -def _task_groups(root: str | Path) -> list[tuple[str, list[HarborTask]]]: - resolved = Path(root).resolve() - dataset_name = resolved.parent.name if is_harbor_task(resolved) else resolved.name - dirs = task_dirs(resolved) - if not dirs: - raise ValueError(f"no Harbor tasks found in {root}") - - groups: dict[tuple[str, str], list[HarborTask]] = {} - for task in (HarborTask.read(task_dir) for task_dir in dirs): - groups.setdefault((task.environment_hash, task.workspace_key), []).append(task) - base_name = slugify(dataset_name) - return sorted( - ( - f"{base_name}-{_group_digest(environment_hash, workspace_key)}", - group, - ) - for (environment_hash, workspace_key), group in groups.items() - ) - - -def _group_digest(environment_hash: str, workspace_key: str) -> str: - """Short stable digest of one group's build and workspace contract.""" - return hashlib.sha256(f"{environment_hash}\0{workspace_key}".encode()).hexdigest()[:12] - - -# ─── task-dir primitives ──────────────────────────────────────────────── - - -def slugify(name: str) -> str: - """A valid env name from a dataset dir name — the SDK's own normalizer, - so a row's env name matches what a deploy of the same context registers.""" - return normalize_environment_name(name, default="harbor") - - -def is_harbor_task(path: Path) -> bool: - """A ``task.toml`` plus an instruction: at the root for a single-step - task, or per ``[[steps]]`` entry for a multi-step one.""" - if not path.is_dir() or not (path / "task.toml").exists(): - return False - if (path / "instruction.md").is_file(): - return True - try: # a multi-step task is only recognizable from its config - return bool(TaskConfig.read(path).steps) - except ValueError: - return False - - -def task_dirs(path: str | Path) -> list[Path]: - """The task dirs under *path*: itself when it is one, else its children.""" - root = Path(path) - if is_harbor_task(root): - return [root] - if root.is_dir(): - return sorted(d for d in root.iterdir() if d.is_dir() and is_harbor_task(d)) - return [] - - -def hash_directory(path: Path) -> str: - """Content-hash a directory for grouping tasks by identical environments.""" - hasher = hashlib.sha256() - if not path.exists(): - return "empty" - for entry in sorted(path.rglob("*")): - name = str(entry.relative_to(path)).encode() - if entry.is_symlink(): - # The link itself is the content. Reading through it would make - # the hash depend on host state outside the context — and could - # walk out of it entirely. - hasher.update(name) - hasher.update(b"\0symlink\0") - hasher.update(os.readlink(entry).encode()) - elif entry.is_file(): - hasher.update(name) - hasher.update(entry.read_bytes()) - return hasher.hexdigest()[:16] diff --git a/integrations/harbor/_runtime.py b/integrations/harbor/_runtime.py deleted file mode 100644 index 3e5827693..000000000 --- a/integrations/harbor/_runtime.py +++ /dev/null @@ -1,388 +0,0 @@ -"""Runtime for an adapted Harbor image.""" - -from __future__ import annotations - -import asyncio -import contextlib -import json -import logging -import math -import os -import pwd -import shutil -from collections.abc import AsyncGenerator, Awaitable, Callable # noqa: TC003 -from pathlib import Path -from typing import Any - -from hud.environment import Environment, Mount -from hud.environment.workspace import Workspace, install_identity_map -from hud.utils.process import ProcessGroup, create_process_group_exec - -from ._adapt import _copy_task_content, _validated_user -from ._load import DEFAULT_VERIFIER_TIMEOUT, HUD_ROOT, HarborTask, TaskConfig, slugify - -LOGGER = logging.getLogger(__name__) - -LOGS = Path("/logs") -VERIFIER_LOGS = LOGS / "verifier" -TESTS = Path("/tests") - - -def phase_uid(task: HarborTask | Path, role: str) -> int | None: - """Resolve the identity for one Harbor phase inside the adapted image.""" - if not isinstance(task, HarborTask): - task = HarborTask.read(task) - user = _validated_user(task.config.phase_user(role), _image_user()) - if user is None: - return None - name = user.split(":", 1)[0] - if name.isdigit(): - return int(name) - try: - return pwd.getpwnam(name).pw_uid - except KeyError as error: - raise ValueError(f"declared user {user!r} does not exist in this image") from error - - -def phase_home(uid: int | None) -> str | None: - """Return the home directory for a phase that drops to uid.""" - if uid is None: - return None - with contextlib.suppress(KeyError): - return pwd.getpwuid(uid).pw_dir - return None - - -def _image_user() -> str | None: - """Read the final image USER recorded by the adaptation layer.""" - with contextlib.suppress(OSError): - return (HUD_ROOT / "image-user").read_text(encoding="utf-8").strip() or None - return None - - -def _harness_out_of_view() -> tuple[Mount, ...]: - """Hide the adapted harness tree from agent sessions.""" - parent = HUD_ROOT.parent - mounts = [Mount("tmpfs", dst=str(parent))] - if parent.is_dir(): - mounts += [ - Mount("rw", src=str(sibling), dst=str(sibling)) - for sibling in sorted(parent.iterdir()) - if sibling != HUD_ROOT - ] - return tuple(mounts) - - -def environment(ref: str | Path = HUD_ROOT / "tasks", *, name: str | None = None) -> Environment: - """Serve the task directories baked into an adapted Harbor image.""" - root = Path(ref) - task_dirs = sorted(d for d in root.iterdir() if d.is_dir()) - if not task_dirs: - raise ValueError(f"no Harbor tasks under {root}") - - tasks = [HarborTask.read(task_dir) for task_dir in task_dirs] - source = tasks[0] - workdir = Path.cwd() - rooted_at_filesystem = workdir == Path("/") - if rooted_at_filesystem: - LOGGER.warning( - "%s has no WORKDIR and declares no [environment] workdir; serving from / " - "with file tracking disabled", - root, - ) - - agent_uid = phase_uid(source, "agent") - env = Environment(name or slugify(root.name)) - workspace = env.workspace( - workdir, - guest_path=workdir.as_posix(), - system_mounts=( - Mount("rw", src="/", dst="/"), - Mount("proc", dst="/proc"), - Mount("dev", dst="/dev"), - ), - mounts=_harness_out_of_view(), - credentials_dir=HUD_ROOT / "session-keys", - shell_uid=agent_uid, - hand_over_root=False, - track_files=False if rooted_at_filesystem else None, - env={ - **({"HOME": home} if (home := phase_home(agent_uid)) else {}), - **source.config.agent.env, - }, - network=source.config.network("agent"), - allowed_hosts=source.config.allowed_hosts("agent"), - require_isolation=True, - ) - - for task in tasks: - _register(env, task, workdir, workspace) - return env - - -def _register(env: Environment, task: HarborTask, workdir: Path, workspace: Workspace) -> None: - config = task.config - - @env.template( - id=task.path.name, - description=config.task.description or f"Harbor task {task.path.name}", - ) - async def _run_harbor_task() -> AsyncGenerator[Any, Any]: - _hide_grading_dirs() - try: - answer = yield (task.path / "instruction.md").read_text(encoding="utf-8") - yield await _grade(task, workdir, answer, workspace) - finally: - _hide_grading_dirs() - await workspace.discard_sandbox() - - -def _sync_tests(task: HarborTask) -> None: - """Leave /tests holding exactly the task's tests.""" - _reset_dir(TESTS) - for child in (task.path / "tests").iterdir(): - _copy_task_content(child, TESTS / child.name) - - -def _hide_grading_dirs() -> None: - """Remove the verifier's tests and verdict from the agent's view.""" - for path in (TESTS, VERIFIER_LOGS): - try: - shutil.rmtree(path) - except FileNotFoundError: - continue - except OSError: - LOGGER.debug("could not remove %s; emptying it instead", path) - _reset_dir(path) - - -def _reset_dir(path: Path) -> None: - """Leave path as an existing, empty directory.""" - if path.is_symlink() or path.is_file(): - path.unlink() - if not path.is_dir(): - path.mkdir(parents=True) - return - for child in path.iterdir(): - if child.is_symlink() or child.is_file(): - child.unlink() - else: - shutil.rmtree(child) - - -async def _grade( - task: HarborTask, workdir: Path, answer: Any, workspace: Workspace | None = None -) -> dict[str, Any]: - _sync_tests(task) - test_sh = TESTS / "test.sh" - test_sh.chmod(test_sh.stat().st_mode | 0o111) - command = [str(test_sh)] - verifier_command = command - - config = task.config - verifier_env: dict[str, str] = {} - verifier_uid = phase_uid(task, "verifier") - setpriv = shutil.which("setpriv") - if verifier_uid is not None and setpriv is not None and os.geteuid() == 0: - _reset_dir(VERIFIER_LOGS) - for root_path in (TESTS, VERIFIER_LOGS): - for target in (root_path, *root_path.rglob("*")): - os.lchown(target, verifier_uid, verifier_uid) - if home := phase_home(verifier_uid): - verifier_env["HOME"] = home - verifier_command = [ - setpriv, - "--reuid", - str(verifier_uid), - "--regid", - str(verifier_uid), - "--clear-groups", - *command, - ] - - severed = not config.network("verifier") - verifier_workspace = workspace - if severed and verifier_workspace is None: - verifier_workspace = Workspace( - Path("/"), - system_mounts=(Mount("proc", dst="/proc"), Mount("dev", dst="/dev")), - network=False, - allowed_hosts=(), - guest_path="/", - hand_over_root=False, - ) - if severed and (verifier_workspace is None or not verifier_workspace.bwrap_available): - raise RuntimeError( - "the verifier declares no-network but bwrap cannot sandbox here; " - "refusing to grade with network access the task ruled out" - ) - - spawn = verifier_command - - async def run_tests() -> ProcessGroup: - environ = {**os.environ, **verifier_env, **config.verifier.env} - if not severed: - return await create_process_group_exec( - *spawn, - cwd=workdir, - env=environ, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - assert verifier_workspace is not None - info_read, info_write = os.pipe() - block_read, block_write = os.pipe() - try: - os.set_inheritable(info_write, True) - os.set_inheritable(block_read, True) - group = await create_process_group_exec( - *verifier_workspace.bwrap_argv( - spawn, - cwd=workdir.as_posix(), - env={**verifier_env, **config.verifier.env}, - inherit_workspace_env=False, - info_fd=info_write, - userns_block_fd=block_read, - network=False, - mount_hosts=False, - isolate_processes=False, - ), - cwd=workdir, - env=environ, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - pass_fds=(info_write, block_read), - ) - os.close(info_write) - os.close(block_read) - info_write = block_read = -1 - await install_identity_map(info_read, block_write) - return group - finally: - os.close(info_read) - os.close(block_write) - for stray in (info_write, block_read): - if stray != -1: - os.close(stray) - - if workspace is None or severed or not workspace.owns_netns: - return await _grade_with_verifier(config, LOGS, answer, run_tests) - sandbox = await workspace.sandbox_pid() - if sandbox is None: - return await _grade_with_verifier(config, LOGS, answer, run_tests) - - async with workspace.visiting(config.allowed_hosts("verifier")) as visitor_env: - verifier_env.update(visitor_env) - spawn = workspace.enter_argv( - sandbox, - command, - env={**verifier_env, **config.verifier.env}, - identity=verifier_uid, - inherit_workspace_env=False, - preserve_credentials=True, - no_new_privs=False, - ) - return await _grade_with_verifier(config, LOGS, answer, run_tests) - - -async def _grade_with_verifier( - config: TaskConfig, - logs: Path, - answer: Any, - run_tests: Callable[[], Awaitable[ProcessGroup]], -) -> dict[str, Any]: - """Run the verifier and shape its reward into a HUD grade.""" - timeout = config.verifier.timeout_sec or DEFAULT_VERIFIER_TIMEOUT - _reset_dir(logs / "verifier") - _write_no_follow(logs / "agent_answer.txt", "" if answer is None else str(answer)) - group = await run_tests() - proc = group.process - assert proc.stdout is not None and proc.stderr is not None - reading = ( - asyncio.create_task(proc.stdout.read()), - asyncio.create_task(proc.stderr.read()), - ) - try: - try: - await asyncio.wait_for(proc.wait(), timeout=timeout) - timed_out = False - except TimeoutError: - timed_out = True - await group.terminate() - out_bytes, err_bytes = await asyncio.gather(*reading) - except BaseException: - for reader in reading: - reader.cancel() - await group.terminate() - raise - if timed_out: - return { - "score": 0.0, - "isError": True, - "content": f"Harbor verifier timed out after {timeout:.0f}s", - "info": { - "verifier_timeout_sec": timeout, - "stdout": out_bytes.decode("utf-8", "replace")[-4000:], - "stderr": err_bytes.decode("utf-8", "replace")[-4000:], - }, - } - - reward, info = _read_reward(logs / "verifier") - info.update( - { - "stdout": out_bytes.decode("utf-8", "replace")[-4000:], - "stderr": err_bytes.decode("utf-8", "replace")[-4000:], - } - ) - if reward is None: - return { - "score": 0.0, - "isError": True, - "content": "Harbor verifier did not write reward.json or reward.txt", - "info": info, - } - return {"score": reward, "info": info} - - -def _read_reward(verifier_logs: Path) -> tuple[float | None, dict[str, Any]]: - """Read a finite reward from reward.json or reward.txt.""" - reward_json = verifier_logs / "reward.json" - if reward_json.is_file(): - try: - data = json.loads(reward_json.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return None, {"parse_error": "reward.json is not valid JSON"} - if (score := _as_score(data)) is not None: - return score, {"reward_file": str(reward_json)} - if isinstance(data, dict): - for key in ("reward", "score"): - if (score := _as_score(data.get(key))) is not None: - return score, {"reward_file": str(reward_json), "reward_json": data} - return None, {"reward_file": str(reward_json), "reward_parse_error": "no numeric reward"} - - reward_txt = verifier_logs / "reward.txt" - if reward_txt.is_file(): - text = reward_txt.read_text(encoding="utf-8").strip() - try: - value = float(text) - except ValueError: - return None, {"reward_file": str(reward_txt), "reward_parse_error": text} - if not math.isfinite(value): - return None, {"reward_file": str(reward_txt), "reward_parse_error": text} - return value, {"reward_file": str(reward_txt)} - - return None, {} - - -def _write_no_follow(path: Path, text: str) -> None: - """Write text without following a symlink at path.""" - if path.is_symlink(): - path.unlink() - with path.open("w", encoding="utf-8") as handle: - handle.write(text) - - -def _as_score(value: Any) -> float | None: - if isinstance(value, bool) or not isinstance(value, int | float): - return None - return float(value) if math.isfinite(float(value)) else None diff --git a/integrations/harbor/adapt.py b/integrations/harbor/adapt.py new file mode 100644 index 000000000..b8e1957b4 --- /dev/null +++ b/integrations/harbor/adapt.py @@ -0,0 +1,330 @@ +"""Build Harbor task directories as runnable HUD environments.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import shutil +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from hud.eval import Task, Taskset +from hud.eval.runtime import RuntimeConfig, RuntimeGPU, RuntimeResources +from hud.utils.docker import docker +from hud.utils.naming import normalize_environment_name + +LOGGER = logging.getLogger(__name__) +ASSETS = Path(__file__).parent +HUD_ROOT = Path("/media/hud") +IGNORED = shutil.ignore_patterns( + "__pycache__", + "*.pyc", + ".git", + ".venv", + "venv", + "*.egg-info", + ".pytest_cache", +) +NetworkMode = Literal["public", "no-network", "allowlist"] + + +class Phase(BaseModel): + model_config = ConfigDict(extra="allow") + + timeout_sec: float | None = Field(default=None, gt=0) + user: str | int | None = None + network_mode: NetworkMode | None = None + allowed_hosts: list[str] = Field(default_factory=list) + env: dict[str, str] = Field(default_factory=dict) + environment: dict[str, Any] | None = None + environment_mode: str | None = None + + +class EnvironmentConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + build_timeout_sec: float = Field(default=600.0, gt=0) + docker_image: str | None = None + os: str = "linux" + cpus: float | None = Field(default=None, gt=0) + memory_mb: int | None = Field(default=None, gt=0) + storage_mb: int | None = Field(default=None, gt=0) + gpus: int | None = Field(default=None, ge=0) + gpu_types: list[str] = Field(default_factory=list) + tpu: dict[str, Any] | None = None + network_mode: NetworkMode = "public" + allowed_hosts: list[str] = Field(default_factory=list) + workdir: str | None = None + env: dict[str, str] = Field(default_factory=dict) + healthcheck: dict[str, Any] | None = None + mcp_servers: list[dict[str, Any]] = Field(default_factory=list) + skills_dir: str | None = None + + +class PackageInfo(BaseModel): + model_config = ConfigDict(extra="allow") + + name: str | None = None + description: str = "" + keywords: list[str] = Field(default_factory=list) + + +class TaskConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + schema_version: str | None = None + task: PackageInfo = Field(default_factory=PackageInfo) + metadata: dict[str, Any] = Field(default_factory=dict) + environment: EnvironmentConfig = Field(default_factory=EnvironmentConfig) + agent: Phase = Field(default_factory=Phase) + verifier: Phase = Field(default_factory=Phase) + steps: list[dict[str, Any]] | None = None + + +@dataclass(frozen=True, slots=True) +class HarborTask: + path: Path + config: TaskConfig + environment_hash: str + + +async def adapt( + path: str | Path, + *, + push: str | None = None, + hud_requirement: str = "hud", +) -> Taskset: + """Build a runnable HUD image for each distinct Harbor environment.""" + root = Path(path).resolve() + if (root / "task.toml").is_file(): + task_dirs = [root] + dataset = root.parent + elif root.is_dir(): + task_dirs = sorted(child for child in root.iterdir() if (child / "task.toml").is_file()) + dataset = root + else: + task_dirs = [] + dataset = root + if not task_dirs: + raise ValueError(f"no Harbor tasks found in {path}") + + tasks = [] + for task_dir in task_dirs: + try: + config = TaskConfig.model_validate( + tomllib.loads((task_dir / "task.toml").read_text("utf-8")) + ) + except (OSError, tomllib.TOMLDecodeError, ValidationError) as error: + raise ValueError( + f"{task_dir.name}/task.toml is not a valid Harbor task: {error}" + ) from error + + unsupported = [] + if config.environment.os != "linux": + unsupported.append(f"os={config.environment.os!r}") + if config.environment.tpu: + unsupported.append("TPUs") + if config.environment.healthcheck: + unsupported.append("healthcheck") + if config.environment.mcp_servers: + unsupported.append("MCP servers") + if config.environment.skills_dir: + unsupported.append("skills_dir") + if config.verifier.environment_mode == "separate" or config.verifier.environment: + unsupported.append("a separate verifier environment") + if config.steps: + unsupported.append("multi-step tasks") + if any( + (task_dir / "environment" / filename).is_file() + for filename in ( + "compose.yaml", + "compose.yml", + "docker-compose.yaml", + "docker-compose.yml", + ) + ): + unsupported.append("Docker Compose") + if unsupported: + raise NotImplementedError( + f"Harbor task {task_dir.name!r} uses unsupported features: " + + ", ".join(unsupported) + ) + + tasks.append( + HarborTask( + path=task_dir, + config=config, + environment_hash=_tree_hash(task_dir / "environment"), + ) + ) + + grouped: dict[tuple[str, str], list[HarborTask]] = {} + for task in tasks: + config = task.config + runtime = json.dumps( + { + "image": config.environment.docker_image, + "workdir": config.environment.workdir, + "environment_env": config.environment.env, + "environment_network": config.environment.network_mode, + "environment_hosts": config.environment.allowed_hosts, + "agent": config.agent.model_dump( + include={"user", "network_mode", "allowed_hosts", "env"} + ), + "verifier": config.verifier.model_dump( + include={"user", "network_mode", "allowed_hosts", "env"} + ), + }, + sort_keys=True, + ) + grouped.setdefault((task.environment_hash, runtime), []).append(task) + + rows = [] + base_name = normalize_environment_name(dataset.name, default="harbor") + for (environment_hash, runtime), group in sorted(grouped.items()): + digest = hashlib.sha256((environment_hash + "\0" + runtime).encode()).hexdigest()[:12] + name = f"{base_name}-{digest}" + source = group[0] + dockerfile = source.path / "environment" / "Dockerfile" + base_image = source.config.environment.docker_image + if base_image and not dockerfile.is_file(): + await docker("pull", base_image) + elif dockerfile.is_file(): + base_image = f"hud-harbor-base:{source.environment_hash}" + await docker( + "build", + "--tag", + base_image, + str(dockerfile.parent), + deadline=max(task.config.environment.build_timeout_sec for task in group), + ) + else: + raise FileNotFoundError(f"{source.path.name} has no environment/Dockerfile") + + output, _ = await docker( + "image", + "inspect", + "--format", + "{{json .Config}}", + base_image, + ) + image_config = json.loads(output) + context = dataset / ".hud-adapt" / name + if context.exists(): + shutil.rmtree(context) + (context / "tasks").mkdir(parents=True) + (context / "packages").mkdir() + for asset in ("Dockerfile", "env.py", "install.sh"): + shutil.copy2(ASSETS / asset, context / asset) + + environment = source.config.environment + workdir = environment.workdir or image_config.get("WorkingDir") or "/" + if Path(workdir).is_relative_to(HUD_ROOT): + raise ValueError(f"Harbor workdir {workdir!r} is inside reserved path {HUD_ROOT}") + manifest = { + "name": name, + "workdir": workdir, + "image_user": image_config.get("User") or None, + "environment": { + "env": environment.env, + "network_mode": environment.network_mode, + "allowed_hosts": environment.allowed_hosts, + }, + "agent": source.config.agent.model_dump( + include={"user", "network_mode", "allowed_hosts", "env"} + ), + "verifier": source.config.verifier.model_dump( + include={"user", "network_mode", "allowed_hosts", "env"} + ), + "tasks": [], + } + for task in group: + if not (task.path / "instruction.md").is_file(): + raise FileNotFoundError(f"{task.path.name} has no instruction.md") + target = context / "tasks" / task.path.name + target.mkdir() + shutil.copy2(task.path / "instruction.md", target / "instruction.md") + shutil.copytree(task.path / "tests", target / "tests", symlinks=True, ignore=IGNORED) + manifest["tasks"].append( + { + "id": task.path.name, + "description": task.config.task.description, + "verifier_timeout": task.config.verifier.timeout_sec or 600.0, + } + ) + (context / "tasks.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + wheel = Path(hud_requirement) + requirement = hud_requirement + if wheel.suffix == ".whl" and wheel.is_file(): + shutil.copy2(wheel, context / "packages" / wheel.name) + requirement = f"{HUD_ROOT}/packages/{wheel.name}" + + tag = _tree_hash(context) + image = f"{push}/{name}:{tag}" if push else f"hud-harbor:{name}-{tag}" + await docker( + "build", + "--build-arg", + f"BASE_IMAGE={base_image}", + "--build-arg", + f"HUD_REQUIREMENT={requirement}", + "--tag", + image, + str(context), + ) + if push: + await docker("push", image) + + for task in group: + config = task.config + resources = RuntimeResources( + cpu=config.environment.cpus, + memory_mb=config.environment.memory_mb, + gpu=( + RuntimeGPU( + count=config.environment.gpus, + type=next(iter(filter(None, config.environment.gpu_types)), None), + ) + if config.environment.gpus + else None + ), + ) + columns = dict(config.metadata) + if config.task.keywords: + columns.setdefault("keywords", config.task.keywords) + rows.append( + Task( + env=name, + id=task.path.name, + columns=columns or None, + runtime_config=RuntimeConfig( + image=image, + resources=resources if resources.model_dump(exclude_none=True) else None, + ), + ) + ) + + LOGGER.info("adapted %d Harbor image(s)", len({task.env for task in rows})) + return Taskset(dataset.name, rows, origin=f"harbor:{dataset}") + + +def _tree_hash(path: Path) -> str: + digest = hashlib.sha256() + if not path.exists(): + return "missing" + for entry in sorted(path.rglob("*")): + name = entry.relative_to(path).as_posix().encode() + if entry.is_symlink(): + digest.update(name + b"\0symlink\0" + os.readlink(entry).encode()) + elif entry.is_file(): + digest.update(name + b"\0" + entry.read_bytes()) + return digest.hexdigest()[:16] diff --git a/integrations/harbor/env.py b/integrations/harbor/env.py new file mode 100644 index 000000000..e6df4703c --- /dev/null +++ b/integrations/harbor/env.py @@ -0,0 +1,307 @@ +"""HUD environment served by every adapted Harbor image.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import math +import os +import pwd +import shutil +from collections.abc import AsyncGenerator # noqa: TC003 +from pathlib import Path +from typing import Any + +from hud.environment import Environment, Mount +from hud.environment.egress import ANY_HOST +from hud.environment.workspace import install_identity_map +from hud.graders import EvaluationResult +from hud.utils.process import ProcessGroup, create_process_group_exec + +ROOT = Path("/media/hud") +TESTS = Path("/tests") +LOGS = Path("/logs") +VERIFIER_LOGS = LOGS / "verifier" +CONFIG = json.loads((ROOT / "tasks.json").read_text("utf-8")) + +os.environ.update(CONFIG["environment"]["env"]) +WORKDIR = Path(CONFIG["workdir"]) +os.chdir(WORKDIR) + + +def network(phase: dict[str, Any]) -> tuple[bool, frozenset[str]]: + baseline = CONFIG["environment"] + mode = phase.get("network_mode") or baseline["network_mode"] + hosts = phase.get("allowed_hosts") if phase.get("network_mode") else baseline["allowed_hosts"] + if mode == "no-network": + return False, frozenset() + if mode == "allowlist": + return True, frozenset(hosts or []) + return True, frozenset({ANY_HOST}) + + +def uid(phase: dict[str, Any]) -> int | None: + declared = phase.get("user") + user = str(declared if declared is not None else CONFIG.get("image_user") or "") + if not user or user in {"root", "0", "root:root", "0:0"}: + return None + name = user.split(":", 1)[0] + if name.isdigit(): + return int(name) + try: + return pwd.getpwnam(name).pw_uid + except KeyError as error: + raise ValueError(f"Harbor user {user!r} does not exist in this image") from error + + +def home(user_id: int | None) -> str | None: + if user_id is None: + return None + with contextlib.suppress(KeyError): + return pwd.getpwuid(user_id).pw_dir + return None + + +def harness_mounts() -> tuple[Mount, ...]: + parent = ROOT.parent + siblings = ( + [ + Mount("rw", src=str(path), dst=str(path)) + for path in sorted(parent.iterdir()) + if path != ROOT + ] + if parent.is_dir() + else [] + ) + return (Mount("tmpfs", dst=str(parent)), *siblings) + + +agent = CONFIG["agent"] +agent_uid = uid(agent) +agent_network, agent_hosts = network(agent) +rooted_at_filesystem = len(WORKDIR.parts) == 1 + +env = Environment(CONFIG["name"]) +workspace = env.workspace( + WORKDIR, + guest_path=WORKDIR.as_posix(), + system_mounts=( + Mount("rw", src="/", dst="/"), + Mount("proc", dst="/proc"), + Mount("dev", dst="/dev"), + ), + mounts=harness_mounts(), + credentials_dir=ROOT / "session-keys", + shell_uid=agent_uid, + hand_over_root=False, + track_files=False if rooted_at_filesystem else None, + env={ + **CONFIG["environment"]["env"], + **agent["env"], + **({"HOME": agent_home} if (agent_home := home(agent_uid)) else {}), + }, + network=agent_network, + allowed_hosts=agent_hosts, + require_isolation=True, +) + + +def register(task: dict[str, Any]) -> None: + task_dir = ROOT / "tasks" / task["id"] + + @env.template( + id=task["id"], + description=task["description"] or f"Harbor task {task['id']}", + ) + async def run() -> AsyncGenerator[Any, Any]: + clear_grading_files() + try: + answer = yield (task_dir / "instruction.md").read_text("utf-8") + yield await grade(task_dir, task["verifier_timeout"], answer) + finally: + clear_grading_files() + await workspace.discard_sandbox() + + +for task_config in CONFIG["tasks"]: + register(task_config) + + +def clear(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + if not path.is_dir(): + path.mkdir(parents=True) + return + for child in path.iterdir(): + if child.is_symlink() or child.is_file(): + child.unlink() + else: + shutil.rmtree(child) + + +def clear_grading_files() -> None: + for path in (TESTS, VERIFIER_LOGS): + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(path) + + +async def grade(task_dir: Path, timeout_sec: float, answer: Any) -> EvaluationResult: + clear(TESTS) + shutil.copytree(task_dir / "tests", TESTS, symlinks=True, dirs_exist_ok=True) + test_script = TESTS / "test.sh" + test_script.chmod(test_script.stat().st_mode | 0o111) + + clear(VERIFIER_LOGS) + write(LOGS / "agent_answer.txt", "" if answer is None else str(answer)) + + verifier = CONFIG["verifier"] + verifier_uid = uid(verifier) + verifier_env = dict(verifier["env"]) + if verifier_uid is not None: + if os.geteuid() == 0 and shutil.which("setpriv") is None: + raise RuntimeError("setpriv is required to run the Harbor verifier as another user") + for root in (TESTS, VERIFIER_LOGS): + for path in (root, *root.rglob("*")): + os.lchown(path, verifier_uid, verifier_uid) + if verifier_home := home(verifier_uid): + verifier_env["HOME"] = verifier_home + + verifier_network, verifier_hosts = network(verifier) + command = [str(test_script)] + if verifier_network: + sandbox = await workspace.sandbox_pid() + if sandbox is None: + raise RuntimeError("the Harbor verifier requires the workspace sandbox") + async with workspace.visiting(verifier_hosts) as visitor_env: + process = await create_process_group_exec( + *workspace.enter_argv( + sandbox, + command, + env={**verifier_env, **visitor_env}, + identity=verifier_uid, + inherit_workspace_env=False, + preserve_credentials=True, + no_new_privs=False, + ), + cwd=WORKDIR, + env={**os.environ, **verifier_env, **visitor_env}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + execution = await process.complete(max_wait=timeout_sec) + else: + process = await isolated(command, verifier_env, verifier_uid) + execution = await process.complete(max_wait=timeout_sec) + + info: dict[str, Any] = { + "exit_code": execution.returncode, + "stdout": execution.stdout.decode("utf-8", "replace")[-4000:], + "stderr": execution.stderr.decode("utf-8", "replace")[-4000:], + } + if execution.timed_out: + info["verifier_timeout_sec"] = timeout_sec + return EvaluationResult( + isError=True, + content=f"Harbor verifier timed out after {timeout_sec:.0f}s", + info=info, + ) + + score, reward_info = reward() + info.update(reward_info) + if score is None: + return EvaluationResult( + isError=True, + content="Harbor verifier did not write a numeric reward", + info=info, + ) + return EvaluationResult(reward=score, info=info) + + +async def isolated( + command: list[str], + verifier_env: dict[str, str], + verifier_uid: int | None, +) -> ProcessGroup: + info_read, info_write = os.pipe() + block_read, block_write = os.pipe() + try: + os.set_inheritable(info_write, True) + os.set_inheritable(block_read, True) + if verifier_uid is not None: + command = [ + shutil.which("setpriv") or "setpriv", + "--reuid", + str(verifier_uid), + "--regid", + str(verifier_uid), + "--clear-groups", + "--", + *command, + ] + process = await create_process_group_exec( + *workspace.bwrap_argv( + command, + cwd=WORKDIR.as_posix(), + env=verifier_env, + inherit_workspace_env=False, + info_fd=info_write, + userns_block_fd=block_read, + network=False, + mount_hosts=False, + isolate_processes=False, + ), + cwd=WORKDIR, + env={**os.environ, **verifier_env}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + pass_fds=(info_write, block_read), + ) + os.close(info_write) + os.close(block_read) + info_write = block_read = -1 + await install_identity_map(info_read, block_write) + return process + finally: + os.close(info_read) + os.close(block_write) + for descriptor in (info_write, block_read): + if descriptor != -1: + os.close(descriptor) + + +def reward() -> tuple[float | None, dict[str, Any]]: + reward_json = VERIFIER_LOGS / "reward.json" + if reward_json.is_file(): + try: + data = json.loads(reward_json.read_text("utf-8")) + except json.JSONDecodeError: + return None, {"reward_parse_error": "reward.json is not valid JSON"} + candidates = [data] + if isinstance(data, dict): + candidates.extend((data.get("reward"), data.get("score"))) + for value in candidates: + if isinstance(value, int | float) and not isinstance(value, bool): + score = float(value) + if math.isfinite(score): + return score, {"reward_file": str(reward_json)} + return None, {"reward_parse_error": "reward.json has no numeric reward"} + + reward_text = VERIFIER_LOGS / "reward.txt" + if reward_text.is_file(): + text = reward_text.read_text("utf-8").strip() + try: + score = float(text) + except ValueError: + score = math.nan + if math.isfinite(score): + return score, {"reward_file": str(reward_text)} + return None, {"reward_parse_error": text} + return None, {} + + +def write(path: Path, text: str) -> None: + if path.is_symlink(): + path.unlink() + path.write_text(text, encoding="utf-8") diff --git a/integrations/harbor/export.py b/integrations/harbor/export.py new file mode 100644 index 000000000..9f175f2d7 --- /dev/null +++ b/integrations/harbor/export.py @@ -0,0 +1,238 @@ +"""Export HUD tasks to self-contained Harbor task folders.""" + +from __future__ import annotations + +import json +import shlex +import shutil +from pathlib import Path +from typing import Any + +from hud.environment import Environment +from hud.environment.server import TaskRunner +from hud.eval import Taskset +from hud.utils.naming import normalize_environment_name + +ALLOWED_PROTOCOLS = ("ssh", "mcp") +DEFAULT_ANSWER_FILE = "/workspace/answer.txt" +CONTROL_PORT = 8765 +BUILD_CONTEXT_IGNORE = shutil.ignore_patterns( + "__pycache__", "*.pyc", ".git", ".venv", "venv", "*.egg-info", ".pytest_cache" +) + +ENTRYPOINT_SH = """\ +#!/bin/sh +set -u + +hud serve {serve_target} --port {port} & + +hud task start {task} --args {args_json} --url tcp://127.0.0.1:{port} || {{ + echo "hud: task setup failed; refusing to run the agent against an unset task" >&2 + exit 1 +}} + +exec "$@" +""" + +TEST_SH = """\ +#!/bin/sh +set -u +mkdir -p /logs/verifier + +ANSWER_FILE={answer_file} +[ -f "$ANSWER_FILE" ] || : > "$ANSWER_FILE" + +if ! hud task grade {task} --args {args_json} --answer-file "$ANSWER_FILE" \\ + --url tcp://127.0.0.1:{port} > /logs/verifier/reward.txt 2> /logs/verifier/grade.err; then + rm -f /logs/verifier/reward.txt + echo "hud: grading failed; see /logs/verifier/grade.err" >&2 + exit 1 +fi +""" + + +async def export( + source: str, + out_dir: str | Path, + *, + answer_file: str = DEFAULT_ANSWER_FILE, + timeout_sec: float = 600.0, +) -> list[Path]: + """Export HUD tasks from *source* into Harbor task folders under *out_dir*. + + ``source`` is a Python task source or a JSON/JSONL taskset next to its + authored environment and Dockerfile. Each task becomes one self-contained + Harbor task folder. + """ + from hud.utils.modules import iter_modules + + src = Path(source).resolve() + source_dir = src.parent if src.is_file() else src + out = Path(out_dir).resolve() + out.mkdir(parents=True, exist_ok=True) + tasks = list(Taskset.from_file(src)) + + scan = source_dir if src.suffix in (".json", ".jsonl") else src + authored: dict[str, Environment] = {} + serve_targets: dict[str, str] = {} + for module in iter_modules(scan): + module_file = Path(getattr(module, "__file__", "") or "") + for attribute, value in vars(module).items(): + if isinstance(value, Environment): + authored[value.name] = value + serve_targets[value.name] = f"{module_file.stem or 'env'}:{attribute}" + + dockerfile = next( + ( + source_dir / name + for name in ("Dockerfile.hud", "Dockerfile") + if (source_dir / name).is_file() + ), + None, + ) + if dockerfile is None: + raise FileNotFoundError( + f"no Dockerfile(.hud) next to {source_dir}; harbor export needs the env's " + "build context to rebuild the image under Harbor.", + ) + + out_resolved = out.resolve() + + def ignore_export(dirpath: str, names: list[str]) -> set[str]: + ignored = set(BUILD_CONTEXT_IGNORE(dirpath, names)) + base = Path(dirpath) + ignored.update(name for name in names if (base / name).resolve() == out_resolved) + return ignored + + created: list[Path] = [] + claimed: dict[str, str] = {} + started: set[str] = set() + try: + for task in tasks: + env = authored.get(task.env) + if env is None or task.id not in env.tasks: + raise TypeError( + f"harbor export needs a local env defining task {task.id!r} " + f"(an env.py named {task.env!r} next to the tasks); none was found.", + ) + if env.name not in started: + started.add(env.name) + await env.start() + + unsupported = [ + capability.protocol + for capability in env.capabilities + if capability.protocol.split("/", 1)[0] not in ALLOWED_PROTOCOLS + ] + if unsupported: + raise ValueError( + f"env {env.name!r} declares non-Harbor capabilities {unsupported}; " + f"only {'/'.join(ALLOWED_PROTOCOLS)} are convertible.", + ) + + declared = task.slug + if not any(character.isalnum() for character in declared): + raise ValueError(f"task slug {declared!r} does not form a usable directory name") + slug = normalize_environment_name( + declared.replace("/", "-").replace("\\", "-"), + default="harbor", + ) + if slug in claimed: + raise ValueError( + f"task slugs {claimed[slug]!r} and {declared!r} both name the export " + f"directory {slug!r}; give them distinct slugs" + ) + claimed[slug] = declared + + task_dir = out / slug + tests_dir = task_dir / "tests" + tests_dir.mkdir(parents=True, exist_ok=True) + + runner = TaskRunner(env.tasks[task.id], task.args) + try: + payload = await runner.start() + finally: + await runner.cancel() + prompt: Any = payload.get("prompt") + if not isinstance(prompt, str): + prompt = json.dumps(prompt, indent=2, default=str) + (task_dir / "instruction.md").write_text( + prompt + + f"\n\n---\nWhen you have finished, write your final answer to `{answer_file}`.\n", + encoding="utf-8", + newline="\n", + ) + + args_json = json.dumps(task.args) + (task_dir / "task.toml").write_text( + 'version = "1.0"\n' + f"name = {json.dumps(slug)}\n" + "\n[metadata]\n" + f"hud_task = {json.dumps(task.id)}\n" + f"hud_args = {json.dumps(args_json)}\n" + "\n[agent]\n" + f"timeout_sec = {timeout_sec}\n" + "\n[verifier]\n" + f"timeout_sec = {timeout_sec}\n", + encoding="utf-8", + newline="\n", + ) + + env_out = task_dir / "environment" + if env_out.exists(): + shutil.rmtree(env_out) + shutil.copytree(source_dir, env_out, ignore=ignore_export, symlinks=True) + + if src.is_file() and src.suffix in (".json", ".jsonl"): + copied_taskset = env_out / src.name + if copied_taskset.is_file(): + copied_taskset.unlink() + copied_ignore = env_out / ".dockerignore" + if copied_ignore.is_file(): + copied_ignore.write_text( + copied_ignore.read_text("utf-8") + "\n!hud_entrypoint.sh\n", + encoding="utf-8", + newline="\n", + ) + for name in ("Dockerfile.hud", "dockerfile"): + alternate = env_out / name + if alternate.exists(): + alternate.unlink() + + serve_target = serve_targets[env.name] + (env_out / "hud_entrypoint.sh").write_text( + ENTRYPOINT_SH.format( + port=CONTROL_PORT, + serve_target=shlex.quote(serve_target), + task=shlex.quote(task.id), + args_json=shlex.quote(args_json), + ), + encoding="utf-8", + newline="\n", + ) + (env_out / "Dockerfile").write_text( + dockerfile.read_text("utf-8").rstrip() + + "\n\n" + + "# HUD runtime for Harbor; final startup directives override the source image.\n" + + "COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh\n" + + 'ENTRYPOINT ["/hud_entrypoint.sh"]\n' + + 'CMD ["sh", "-c", "sleep infinity"]\n', + encoding="utf-8", + newline="\n", + ) + (tests_dir / "test.sh").write_text( + TEST_SH.format( + port=CONTROL_PORT, + task=shlex.quote(task.id), + args_json=shlex.quote(args_json), + answer_file=shlex.quote(answer_file), + ), + encoding="utf-8", + newline="\n", + ) + created.append(task_dir) + finally: + for name in started: + await authored[name].stop() + + return created diff --git a/integrations/harbor/install.sh b/integrations/harbor/install.sh new file mode 100644 index 000000000..a24cdfc55 --- /dev/null +++ b/integrations/harbor/install.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +requirement="${1:-hud}" +root=/media/hud +python_version=3.12 + +export UV_INSTALL_DIR="$root/bin" +export UV_PYTHON_INSTALL_DIR="$root/python" +export UV_PYTHON_BIN_DIR="$root/bin" +export UV_NO_CACHE=1 +export XDG_CONFIG_HOME="$root/config" +export PATH="$root/bin:$PATH" + +if ! command -v uv >/dev/null 2>&1; then + if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then + { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \ + || apk add --no-cache curl ca-certificates + fi + { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \ + || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \ + || pip install -q -U uv +fi + +command -v bwrap >/dev/null 2>&1 \ + || { apt-get update -qq && apt-get install -y -qq bubblewrap; } \ + || apk add --no-cache bubblewrap + +uv python install "$python_version" +uv venv "$root/venv" --python "$python_version" +uv pip install --python "$root/venv/bin/python" "$requirement" + +apt_packages="" +apk_packages="" +for spec in \ + "python3|python3 python3-venv|python3" \ + "pip3|python3-pip|py3-pip" \ + "git|git|git" \ + "curl|curl ca-certificates|curl ca-certificates" +do + command="${spec%%|*}" + if command -v "$command" >/dev/null 2>&1; then + continue + fi + rest="${spec#*|}" + apt_packages="$apt_packages ${rest%%|*}" + apk_packages="$apk_packages ${rest##*|}" +done + +if [ -n "$apt_packages" ]; then + { apt-get update -qq && apt-get install -y -qq $apt_packages; } \ + || apk add --no-cache $apk_packages +fi diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index 938863e9d..84effe924 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -1,829 +1,291 @@ -"""The Harbor integration as data: load, provenance, adaptation, and runtime. - -Docker-side serving needs a daemon and is covered by the e2e integration -scripts; here ``load``'s rows/provenance/stamping and ``adapt``'s build -contexts are checked without one, exercising the integration directly (no -eval/taskset wiring). -""" +"""Observable contracts for adapting Harbor tasks into HUD images.""" from __future__ import annotations +import importlib +import json import os -import re import subprocess -from types import SimpleNamespace -from typing import TYPE_CHECKING +from pathlib import Path import pytest from integrations import harbor -from integrations.harbor import _load as harbor_load - -if TYPE_CHECKING: - from pathlib import Path - -def _write_harbor_task(root: Path, name: str, marker: str = "FROM python:3.12-slim\n") -> Path: - task = root / name - (task / "environment").mkdir(parents=True) - (task / "tests").mkdir() - (task / "instruction.md").write_text(f"Fix {name}.\n", encoding="utf-8") - (task / "task.toml").write_text( - f'schema_version = "1.3"\n\n[task]\nname = "demo/{name}"\n', encoding="utf-8" - ) - (task / "environment" / "Dockerfile").write_text(marker, encoding="utf-8") - (task / "tests" / "test.sh").write_text( - "#!/usr/bin/env bash\nmkdir -p /logs/verifier\necho 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - return task +from .conftest import make_harbor_task, make_multi_step_task @pytest.fixture(autouse=True) -def stub_docker(monkeypatch) -> None: - async def fake_docker(*args, **kwargs): - return b"", b"" - - monkeypatch.setattr("integrations.harbor._adapt._docker", fake_docker) - +def fake_docker(monkeypatch): + calls: list[tuple[str, ...]] = [] -def test_load_stamps_rows_with_provenance(tmp_path) -> None: - _write_harbor_task(tmp_path, "task-a") - _write_harbor_task(tmp_path, "task-b") + async def run(*args: str, **_kwargs): + calls.append(args) + if args[:3] == ("image", "inspect", "--format"): + return json.dumps({"User": "", "WorkingDir": "/workspace"}), "" + return "", "" - taskset = harbor.load(tmp_path) + module = importlib.import_module("integrations.harbor.adapt") + monkeypatch.setattr(module, "docker", run) + return calls - assert taskset.origin == f"harbor:{tmp_path.resolve()}" - assert len(taskset) == 2 - assert all(t.runtime_config is None for t in taskset) - -async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None: - _write_harbor_task(tmp_path, "task-a") - _write_harbor_task(tmp_path, "task-b", marker="FROM python:3.11-slim\n") +async def test_adapt_builds_the_source_then_an_authored_hud_environment( + tmp_path: Path, + fake_docker, +) -> None: + make_harbor_task(tmp_path, "task-a") taskset = await harbor.adapt(tmp_path) - assert len(taskset) == 2 - assert all(task.runtime_config and task.runtime_config.image for task in taskset) - contexts = sorted(p.name for p in (tmp_path / ".hud-adapt").iterdir()) - assert len(contexts) == 2 # two env groups (distinct Dockerfiles) - context = tmp_path / ".hud-adapt" / contexts[0] - dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - # The CMD serves the contract constructor by module reference — no baked env.py. - assert "harbor:environment" in dockerfile - assert f'"name={context.name}"' in dockerfile - assert "EXPOSE 8765" in dockerfile - assert not (context / "_hud" / "env.py").exists() - baked = context / "_hud" / "tasks" - (task_dir,) = list(baked.iterdir()) - assert (task_dir / "instruction.md").is_file() - assert (task_dir / "tests" / "test.sh").is_file() - - -async def test_the_layer_keeps_its_own_state_out_of_the_task(tmp_path) -> None: - # Everything the layer installs belongs under the harness's own tree: uv's - # binary, its managed interpreter and that interpreter's shims, its cache - # and config. At uv's defaults these land in the invoking user's home — - # inside the task's own filesystem, where the agent finds a HUD runtime the - # task never declared, and shims into a hidden tree resolving to nothing. - _write_harbor_task(tmp_path, "task-a") - await harbor.adapt(tmp_path) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - script = (context / "_hud" / "install.sh").read_text(encoding="utf-8") - - directories = dict(re.findall(r"\b(UV_\w*(?:DIR|HOME)|XDG_CONFIG_HOME)=(\S+)", script)) - assert directories, "the layer pins no uv directories" - assert all(path.startswith(f"{harbor_load.HUD_ROOT}/") for path in directories.values()), ( - directories - ) - # Nothing is written relative to the image's home, so there is no scattered - # state to clean up afterwards. - assert "$HOME" not in script - # The CLI's update check writes into that home too, and calls PyPI on every - # rollout since a fresh container never has a warm cache. - dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - assert "ENV HUD_SKIP_VERSION_CHECK=1" in dockerfile - - -async def test_the_agent_toolchain_installs_only_what_the_image_lacks(tmp_path) -> None: - # Harbor's agent install provisions python3/pip/git/curl into the task - # container before the agent phase; an adapted image bakes the same set at - # build time instead. An image shipping its own interpreter must keep it, - # so every tool is presence-checked rather than installed outright. - _write_harbor_task(tmp_path, "task-a") - await harbor.adapt(tmp_path) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - script = (context / "_hud" / "install.sh").read_text(encoding="utf-8") - decision = script[script.index('apt_pkgs=""') : script.index("\ndone") + len("\ndone")] - - def queued(path: str) -> set[str]: - # Runs under ``set -eu`` as the image does: a tool already present must - # not abort the script mid-build. - result = subprocess.run( - ["/bin/sh", "-c", f'set -eu\nPATH={path}\n{decision}\nprintf "%s" "$apt_pkgs"'], - capture_output=True, - text=True, - check=True, - ) - return set(result.stdout.split()) - - assert queued("/nonexistent") == { - "python3", - "python3-venv", - "python3-pip", - "git", - "curl", - "ca-certificates", - } - - stub = tmp_path / "bin" - stub.mkdir() - for tool in ("python3", "pip3"): - (stub / tool).write_text("#!/bin/sh\n", encoding="utf-8") - (stub / tool).chmod(0o755) - - assert queued(str(stub)) == {"git", "curl", "ca-certificates"} - - -async def test_adapt_binds_built_images_to_rows(tmp_path) -> None: - _write_harbor_task(tmp_path, "task-a") - (task,) = list(await harbor.adapt(tmp_path)) - + (task,) = list(taskset) + assert task.id == "task-a" assert task.runtime_config is not None - assert task.runtime_config.image.startswith("hud-harbor-adapted:") + assert task.runtime_config.image is not None + assert task.runtime_config.image.startswith("hud-harbor:") - (bare,) = list(harbor.load(tmp_path)) - assert bare.runtime_config is None or bare.runtime_config.image is None + builds = [call for call in fake_docker if call[0] == "build"] + assert len(builds) == 2 + assert "BASE_IMAGE=hud-harbor-base:" in " ".join(builds[1]) + (context,) = (tmp_path / ".hud-adapt").iterdir() + integration = Path(__file__).parents[1] + for asset in ("Dockerfile", "env.py", "install.sh"): + assert (context / asset).read_bytes() == (integration / asset).read_bytes() + assert (context / "tasks" / "task-a" / "instruction.md").is_file() + assert (context / "tasks" / "task-a" / "tests" / "test.sh").is_file() -async def test_environment_serves_the_baked_tasks(tmp_path, monkeypatch) -> None: - # The constructor refuses to build an unsandboxed env (hiding the - # harness's tree is an integrity property); tests run outside a - # container, so stub bwrap. - monkeypatch.setattr("hud.environment.workspace.usable_bwrap", lambda: "/usr/bin/true") - _write_harbor_task(tmp_path, "task-a") - _write_harbor_task(tmp_path, "task-b") - await harbor.adapt(tmp_path) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - - env = harbor.environment(context / "_hud" / "tasks", name=context.name) - - assert env.name == context.name - assert sorted(env.tasks) == ["task-a", "task-b"] - # The adapted CMD serves exactly this constructor. - dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - assert "harbor:environment" in dockerfile +async def test_adapt_groups_identical_images_and_keeps_row_metadata( + dataset_same_env: Path, + fake_docker, +) -> None: + taskset = await harbor.adapt(dataset_same_env) + + assert len(taskset) == 3 + assert len(taskset.environment_names()) == 1 + assert all( + task.columns + == { + "category": "systems", + "difficulty": "medium", + "tags": ["bash", "linux"], + } + for task in taskset + ) + assert len([call for call in fake_docker if call[0] == "build"]) == 2 -def test_harbor_implements_the_integration_contract() -> None: - from hud.environment import Integration - assert isinstance(harbor.integration, Integration) - assert harbor.integration.name == "harbor" +async def test_distinct_environments_build_distinct_images( + dataset_multi_env: Path, + fake_docker, +) -> None: + taskset = await harbor.adapt(dataset_multi_env) + assert len(taskset.environment_names()) == 2 + assert len([call for call in fake_docker if call[0] == "build"]) == 4 -def test_public_surface_stays_at_the_integration_boundary() -> None: - assert set(harbor.__all__) == { - "adapt", - "detect", - "environment", - "export", - "integration", - "load", - } - assert not hasattr(harbor, "grouped") - assert not hasattr(harbor, "docker_runtime") - assert not hasattr(harbor, "agent_timeout") - -def test_load_translates_declared_requirements(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") +async def test_adapt_maps_resources_and_pushes_the_images(tmp_path: Path, fake_docker) -> None: + task = make_harbor_task(tmp_path, "gpu") (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - "[agent]\ntimeout_sec = 2400.0\n\n" - "[environment]\ncpus = 4\nmemory_mb = 8192\ngpus = 2\n" - "build_timeout_sec = 600.0\nstorage_mb = 10240\n", + """ +[metadata] +difficulty = "hard" + +[environment] +cpus = 4 +memory_mb = 8192 +gpus = 2 +gpu_types = ["H100"] +""", encoding="utf-8", ) - (row,) = list(harbor.load(tmp_path)) + (row,) = list(await harbor.adapt(tmp_path, push="registry.example/hud")) + assert row.columns == {"difficulty": "hard"} assert row.runtime_config is not None + assert row.runtime_config.image is not None + assert row.runtime_config.image.startswith("registry.example/hud/") assert row.runtime_config.resources is not None - assert row.runtime_config.resources.cpu == 4.0 + assert row.runtime_config.resources.cpu == 4 assert row.runtime_config.resources.memory_mb == 8192 assert row.runtime_config.resources.gpu is not None assert row.runtime_config.resources.gpu.count == 2 - # Time budgets are the engine's, not the substrate's. - assert row.runtime_config.limits is None - - -def test_load_omits_requirements_a_task_does_not_declare(tmp_path) -> None: - _write_harbor_task(tmp_path, "task-a") # minimal task.toml: no resources - - (row,) = list(harbor.load(tmp_path)) - - assert row.runtime_config is None - - -def test_load_carries_metadata_as_columns(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n' - 'description = "Fix the thing properly."\nkeywords = ["shell", "debug"]\n\n' - '[metadata]\ndifficulty = "hard"\ncategory = "systems"\ntags = ["a", "b"]\n', - encoding="utf-8", - ) - - (row,) = list(harbor.load(tmp_path)) - - assert row.columns is not None - assert row.columns["difficulty"] == "hard" - assert row.columns["category"] == "systems" - assert row.columns["keywords"] == ["shell", "debug"] - - -async def test_served_templates_use_the_declared_description(tmp_path, monkeypatch) -> None: - # The constructor refuses to build an unsandboxed env (hiding the - # harness's tree is an integrity property); tests run outside a - # container, so stub bwrap. - monkeypatch.setattr("hud.environment.workspace.usable_bwrap", lambda: "/usr/bin/true") - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n' - 'description = "Fix the thing properly."\n', - encoding="utf-8", - ) - await harbor.adapt(tmp_path) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - - env = harbor.environment(context / "_hud" / "tasks", name=context.name) - - assert env.tasks["task-a"].description == "Fix the thing properly." - - -async def test_multi_step_tasks_load_but_cannot_be_adapted_yet(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[[steps]]\nname = "first"\n', - encoding="utf-8", - ) - (task / "instruction.md").unlink() # multi-step: instructions live per step - - assert [row.id for row in harbor.load(tmp_path)] == ["task-a"] - with pytest.raises(NotImplementedError, match="multi-step"): - await harbor.adapt(tmp_path) + assert row.runtime_config.resources.gpu.type == "H100" + assert any(call[0] == "push" for call in fake_docker) -def test_declared_workspace_contract_is_parsed_once(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") +async def test_prebuilt_harbor_image_skips_the_source_build(tmp_path: Path, fake_docker) -> None: + task = make_harbor_task(tmp_path, "prebuilt", dockerfile=None) (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[environment]\nworkdir = "/srv/app"\n\n' - '[environment.env]\nTOKEN = "abc"\n', - encoding="utf-8", - ) - - config = harbor_load.HarborTask.read(task).config - - assert config.network("agent") is True - assert config.environment.env == {"TOKEN": "abc"} - assert config.agent.env == {} - assert config.environment.workdir == "/srv/app" - assert config.allowed_hosts("agent") == frozenset({"*"}) - assert config.allowed_hosts("verifier") == frozenset({"*"}) - assert config.phase_user("agent") is None - assert config.phase_user("verifier") is None - - -def test_each_phase_is_held_to_the_hosts_it_declared(tmp_path) -> None: - """An allowlist written for one phase must not become the other's. The - agent's applied to the verifier is a grader that cannot install its own - tooling — a zero the task's author never wrote.""" - isolated = _write_harbor_task(tmp_path, "isolated") - (isolated / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/isolated"\n\n' - '[environment]\nnetwork_mode = "no-network"\n', - encoding="utf-8", - ) - isolated_task = harbor_load.HarborTask.read(isolated) - isolated_config = isolated_task.config - assert isolated_task.unsupported_features() == [] - assert isolated_config.network("agent") is False - assert isolated_config.allowed_hosts("agent") == frozenset() - assert isolated_config.allowed_hosts("verifier") == frozenset() - - # Declared for the whole environment: both phases named those hosts. - shared = _write_harbor_task(tmp_path, "shared") - (shared / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/shared"\n\n' - '[environment]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', - encoding="utf-8", - ) - shared_task = harbor_load.HarborTask.read(shared) - shared_config = shared_task.config - assert shared_task.unsupported_features() == [] - assert shared_config.allowed_hosts("agent") == frozenset({"pypi.org"}) - assert shared_config.allowed_hosts("verifier") == frozenset({"pypi.org"}) - - # Declared for the agent alone: the verifier said nothing, so it is public. - agent_only = _write_harbor_task(tmp_path, "agent-only") - (agent_only / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/agent-only"\n\n' - '[agent]\nnetwork_mode = "allowlist"\nallowed_hosts = ["pypi.org"]\n', + '[environment]\ndocker_image = "registry.example/base:latest"\n', encoding="utf-8", ) - agent_task = harbor_load.HarborTask.read(agent_only) - agent_config = agent_task.config - assert agent_task.unsupported_features() == [] - assert agent_config.allowed_hosts("agent") == frozenset({"pypi.org"}) - assert agent_config.allowed_hosts("verifier") == frozenset({"*"}) - -def test_tasks_with_different_policies_get_separate_envs(tmp_path) -> None: - # Same build context, different declared workdir: one env serves one - # policy, so these must not share an environment. - for name, workdir in (("here", "/app"), ("there", "/srv")): - task = _write_harbor_task(tmp_path, name) - (task / "task.toml").write_text( - f'schema_version = "1.3"\n\n[task]\nname = "demo/{name}"\n\n' - f'[environment]\nworkdir = "{workdir}"\n', - encoding="utf-8", - ) - - envs = {row.env for row in harbor.load(tmp_path)} - - assert len(envs) == 2 - - -def test_adapted_cmd_serves_the_contract_constructor(tmp_path) -> None: - import asyncio - - _write_harbor_task(tmp_path, "task-a") - asyncio.get_event_loop_policy() - asyncio.run(harbor.adapt(tmp_path)) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - - assert "harbor:environment" in dockerfile - - -async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> None: - # /logs is agent-reachable: a reward.json planted before grading must not - # out-rank the verifier's own output. - import asyncio - import json - - from integrations.harbor._runtime import _grade_with_verifier - - task = _write_harbor_task(tmp_path, "task-a") - logs = tmp_path / "logs" - (logs / "verifier").mkdir(parents=True) - (logs / "verifier" / "reward.json").write_text(json.dumps({"reward": 1.0}), encoding="utf-8") - - async def run_tests(): - from hud.utils.process import create_process_group_exec - - return await create_process_group_exec( - "bash", - "-c", - f"echo 0 > {logs}/verifier/reward.txt", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - - grade = await _grade_with_verifier(harbor_load.TaskConfig.read(task), logs, None, run_tests) - - assert grade["score"] == 0.0 - - -async def test_a_declared_identity_governs_the_phases_not_the_harness(tmp_path) -> None: - # Harbor runs the agent and the verifier as the identity the task declares - # for each; its harness is not subject to either. Serving from inside the - # image, a USER directive would demote the harness too, leaving it unable - # to create /tests at the filesystem root or its own state under the - # harness tree. - task = _write_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - "FROM python:3.12-slim\nRUN useradd -m agent\nUSER agent\n", encoding="utf-8" - ) await harbor.adapt(tmp_path) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - layer = (context / "Dockerfile").read_text(encoding="utf-8").split("adaptation layer", 1)[1] - - assert "USER root" in layer # installs, and goes on serving, as root - assert "USER agent" not in layer - assert "mkdir -p /tests" not in layer # root makes them when grading needs them - # The image's own declaration is not lost: it is kept for serve time, - # where the build context no longer exists. - assert (context / "_hud" / "image-user").read_text(encoding="utf-8").strip() == "agent" - - -def test_each_phase_takes_the_identity_declared_for_it(tmp_path, monkeypatch) -> None: - # A task may hand the agent a restricted account and still verify as root. - from integrations.harbor import _runtime - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[agent]\nuser = "agent"\n', - encoding="utf-8", - ) - monkeypatch.setattr(_runtime, "_image_user", lambda: None) - monkeypatch.setattr(_runtime.pwd, "getpwnam", lambda name: SimpleNamespace(pw_uid=1000)) - - assert _runtime.phase_uid(task, "agent") == 1000 - assert _runtime.phase_uid(task, "verifier") is None # root, as declared by omission - - -def test_build_stage_entrypoint_does_not_refuse(tmp_path) -> None: - # Adaptation replaces only final-stage startup: an ENTRYPOINT confined to - # a build stage is no reason to refuse the task. - task = _write_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - 'FROM golang:1.22 AS build\nENTRYPOINT ["/tool"]\nRUN true\n' - "FROM python:3.12-slim\nWORKDIR /app\n", - encoding="utf-8", - ) - - assert harbor_load.HarborTask.read(task).unsupported_features() == [] - - -def test_declared_uid_zero_beats_the_source_user(tmp_path) -> None: - # uid 0 is a declaration, not an absence: it must not fall through to the - # Dockerfile's own USER. - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - "[agent]\nuser = 0\n\n[verifier]\nuser = 0\n", - encoding="utf-8", - ) - - assert harbor_load.HarborTask.read(task).config.agent.user == 0 - - -def test_rewards_are_finite_numbers_not_booleans(tmp_path) -> None: - import json as jsonlib + builds = [call for call in fake_docker if call[0] == "build"] + assert len(builds) == 1 + assert "BASE_IMAGE=registry.example/base:latest" in builds[0] + assert ("pull", "registry.example/base:latest") in fake_docker - from integrations.harbor._runtime import _read_reward - logs = tmp_path / "verifier" - logs.mkdir() - for planted in ( - "true", - jsonlib.dumps({"reward": True}), - jsonlib.dumps({"reward": float("inf")}), - ): - (logs / "reward.json").write_text(planted, encoding="utf-8") - score, _info = _read_reward(logs) - assert score is None, planted - (logs / "reward.json").unlink() - (logs / "reward.txt").write_text("nan", encoding="utf-8") - score, _info = _read_reward(logs) - assert score is None - (logs / "reward.txt").write_text("0.75", encoding="utf-8") - assert _read_reward(logs)[0] == 0.75 - - -def test_first_named_gpu_type_is_requested(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[environment]\ngpus = 1\ngpu_types = ["", "H100"]\n', - encoding="utf-8", - ) +async def test_zero_gpus_is_a_valid_harbor_resource_declaration( + tmp_path: Path, + fake_docker, +) -> None: + task = make_harbor_task(tmp_path, "cpu-only") + (task / "task.toml").write_text("[environment]\ngpus = 0\n", encoding="utf-8") - (row,) = list(harbor.load(tmp_path)) + (row,) = list(await harbor.adapt(tmp_path)) assert row.runtime_config is not None - assert row.runtime_config.resources is not None - assert row.runtime_config.resources.gpu is not None - assert row.runtime_config.resources.gpu.type == "H100" + assert row.runtime_config.resources is None -def test_an_invalid_task_toml_is_an_error_not_a_default(tmp_path) -> None: - # Silently falling back to defaults would grade the task under - # requirements its author never wrote. - task = _write_harbor_task(tmp_path, "task-a") +async def test_runtime_configuration_is_data_not_dockerfile_codegen( + tmp_path: Path, + fake_docker, +) -> None: + task = make_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[environment]\ncpus = "lots"\n', - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="not a valid Harbor task"): - harbor.load(tmp_path) - - -def test_grading_directories_do_not_exist_during_the_agent_phase(tmp_path, monkeypatch) -> None: - """The image ships /tests and the verdict dir empty, and a previous rollout - leaves them behind — either way the agent phase must not find them.""" - from integrations.harbor import _runtime + """ +[environment] +workdir = "/app" +network_mode = "allowlist" +allowed_hosts = ["pypi.org"] - tests, verdict = tmp_path / "tests", tmp_path / "logs" / "verifier" - for stale in (tests, verdict): - stale.mkdir(parents=True) - (tests / "test.sh").write_text("the assertions", encoding="utf-8") - monkeypatch.setattr(_runtime, "TESTS", tests) - monkeypatch.setattr(_runtime, "VERIFIER_LOGS", verdict) +[environment.env] +SHARED = "yes" - _runtime._hide_grading_dirs() +[agent] +user = "agent" - assert not tests.exists() - assert not verdict.exists() - # /logs itself is Harbor's and stays: the agent's own answer is written there. - assert verdict.parent.exists() +[agent.env] +AGENT_ONLY = "yes" +[verifier] +user = 0 +network_mode = "no-network" -async def test_masks_are_applied_after_the_workspace_bind(tmp_path, monkeypatch) -> None: - # bwrap applies ``mounts`` after the workspace bind; as system mounts the - # masks would be re-covered when the guest path is "/" (no WORKDIR). - from hud.environment import workspace as workspace_mod - - monkeypatch.setattr(workspace_mod, "usable_bwrap", lambda: "/usr/bin/true") - built: list[workspace_mod.Workspace] = [] - original = workspace_mod.Workspace - - def record(*args, **kwargs): - ws = original(*args, **kwargs) - built.append(ws) - return ws - - monkeypatch.setattr("hud.environment.env.Workspace", record) - _write_harbor_task(tmp_path, "task-a") - await harbor.adapt(tmp_path) - (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) - - harbor.environment(context / "_hud" / "tasks", name=context.name) - - (workspace,) = built - masked = [m.dst for m in workspace.mounts] - # The harness's tree is hidden by rebuilding its parent, so the entry is - # absent rather than present-and-empty: a directory named after the - # harness is the tell, not what is inside it. - assert str(harbor_load.HUD_ROOT.parent) in masked - assert str(harbor_load.HUD_ROOT) not in masked - # The assertions and the verdict are kept from the agent phase by not - # existing during it, not by masking: an empty directory the base image - # does not have is itself a signal. - assert "/tests" not in masked - assert "/logs/verifier" not in masked - assert [m.dst for m in workspace._system_mounts] == ["/", "/proc", "/dev"] - - -async def test_dataset_symlinks_are_never_dereferenced(tmp_path) -> None: - # A dataset is untrusted: a link out of it must stay a link, not pull - # host files into the build context or the served /tests. - secret = tmp_path / "outside" / "secret.txt" - secret.parent.mkdir() - secret.write_text("host-only", encoding="utf-8") - task = _write_harbor_task(tmp_path / "ds", "task-a") - (task / "tests" / "leak.txt").symlink_to(secret) - - await harbor.adapt(tmp_path / "ds") - (context,) = sorted((tmp_path / "ds" / ".hud-adapt").iterdir()) - - # The link is copied as a link: its target was never read, so no host - # content entered the context (inside the image the link simply dangles). - baked = context / "_hud" / "tasks" / "task-a" / "tests" / "leak.txt" - assert baked.is_symlink() - assert os.readlink(baked) == str(secret) - - -def test_one_network_decision_serves_both_phases(tmp_path) -> None: - # Container-wide isolation binds agent and verifier alike; a phase-level - # declaration binds that phase. One function answers for both. - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[environment]\nnetwork_mode = "no-network"\n', - encoding="utf-8", - ) - - assert harbor_load.TaskConfig.read(task).network("agent") is False - assert harbor_load.TaskConfig.read(task).network("verifier") is False - - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[verifier]\nnetwork_mode = "no-network"\n', +[verifier.env] +VERIFIER_ONLY = "yes" +""", encoding="utf-8", ) - assert harbor_load.TaskConfig.read(task).network("agent") is True - assert harbor_load.TaskConfig.read(task).network("verifier") is False - - -def test_final_stage_reads_only_what_the_shipped_image_declares() -> None: - """The one Dockerfile parser, across the shapes that misled it before. - - Each ``FROM`` opens a stage, so a build stage's ``USER``/``ENTRYPOINT`` - is not the shipped image's; ``user[:group]`` is a legal operand; and a - heredoc body is data, not instructions. - """ - from integrations.harbor._load import final_stage + await harbor.adapt(tmp_path) - multistage = final_stage( - 'FROM golang:1.22 AS build\nUSER builder\nENTRYPOINT ["/tool"]\nRUN true\n' - "FROM python:3.12-slim\nWORKDIR /app\nUSER app:app\n" - ) - assert multistage.user == "app:app" # group form preserved - assert "ENTRYPOINT" not in multistage.directives # build stage's, not shipped - - # A build stage's USER alone leaves the shipped stage's identity unset. - assert final_stage("FROM golang AS build\nUSER builder\nFROM python:3.12-slim\n").user is None - - # Root in any spelling is "no declared identity to restore". - assert final_stage("FROM x\nUSER root\n").user is None - assert final_stage("FROM x\nUSER 0:0\n").user is None - - # A heredoc writing another Dockerfile is not this one's instructions. - heredoc = final_stage( - "FROM python:3.12-slim\n" - "RUN < /tmp/generated.Dockerfile\n" - 'FROM someone-else:latest\nUSER hacker\nENTRYPOINT ["/elsewhere"]\n' - "EOF\nUSER app\n" - ) - assert heredoc.user == "app" - assert "ENTRYPOINT" not in heredoc.directives + (context,) = (tmp_path / ".hud-adapt").iterdir() + manifest = json.loads((context / "tasks.json").read_text("utf-8")) + assert manifest["workdir"] == "/app" + assert manifest["environment"] == { + "env": {"SHARED": "yes"}, + "network_mode": "allowlist", + "allowed_hosts": ["pypi.org"], + } + assert manifest["agent"]["user"] == "agent" + assert manifest["agent"]["env"] == {"AGENT_ONLY": "yes"} + assert manifest["verifier"]["user"] == 0 + assert manifest["verifier"]["network_mode"] == "no-network" + assert manifest["verifier"]["env"] == {"VERIFIER_ONLY": "yes"} + dockerfile = (context / "Dockerfile").read_text("utf-8") + assert "SHARED" not in dockerfile + assert "WORKDIR /app" not in dockerfile @pytest.mark.parametrize( ("declaration", "expected"), [ - ('[environment.healthcheck]\ncommand = "curl -sf localhost/health"\n', "healthcheck"), - ( - '[[environment.mcp_servers]]\nname = "db"\nurl = "http://localhost:9000/sse"\n', - "mcp_servers", - ), - ('[verifier]\nenvironment_mode = "separate"\n', "own environment"), - ('[environment]\ntpu = {type = "v5", topology = "2x2"}\n', "tpu"), - ('[environment]\nos = "windows"\n', "os"), - ('[agent]\nuser = "agent"\n\n[verifier]\nuser = "root"\n', "one USER"), - ('[[steps]]\nname = "first"\n', "multi-step"), + ('[environment]\nos = "windows"\n', "os="), + ('[environment]\ntpu = {type = "v5", topology = "2x2"}\n', "TPUs"), + ('[environment.healthcheck]\ncommand = "curl localhost"\n', "healthcheck"), + ('[[environment.mcp_servers]]\nname = "db"\n', "MCP servers"), + ('[verifier]\nenvironment_mode = "separate"\n', "separate verifier"), ], ) -def test_declarations_this_integration_cannot_reproduce_are_refused( - tmp_path, declaration: str, expected: str +async def test_unsupported_harbor_behaviour_fails_before_building( + tmp_path: Path, + fake_docker, + declaration: str, + expected: str, ) -> None: - """A wrong score is worse than a refused task, so each of these names - itself in the refusal rather than being silently dropped.""" - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - f'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n{declaration}', - encoding="utf-8", - ) - - assert expected in " ".join(harbor_load.HarborTask.read(task).unsupported_features()) + task = make_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text(declaration, encoding="utf-8") - -async def test_a_refused_task_never_reaches_a_build_context(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[environment.healthcheck]\ncommand = "curl -sf localhost/health"\n', - encoding="utf-8", - ) - - with pytest.raises(NotImplementedError, match="healthcheck"): + with pytest.raises(NotImplementedError, match=expected): await harbor.adapt(tmp_path) - -async def test_a_chatty_verifier_does_not_deadlock(tmp_path) -> None: - # More output than a pipe buffer holds (~64KB): draining only after exit - # would block the writer forever and score a finished script as a timeout. - import asyncio - - from hud.utils.process import create_process_group_exec - from integrations.harbor._runtime import _grade_with_verifier - - task = _write_harbor_task(tmp_path, "task-a") - logs = tmp_path / "logs" - (logs / "verifier").mkdir(parents=True) - - async def run_tests(): - return await create_process_group_exec( - "bash", - "-c", - f"head -c 400000 /dev/zero | tr '\\0' 'x'; echo 1 > {logs}/verifier/reward.txt", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - - grade = await asyncio.wait_for( - _grade_with_verifier(harbor_load.TaskConfig.read(task), logs, None, run_tests), - timeout=30, - ) - - assert grade["score"] == 1.0 - assert len(grade["info"]["stdout"]) > 0 + assert fake_docker == [] -def test_phase_env_reaches_the_phase_that_declared_it(tmp_path) -> None: - # [environment.env] is container-wide, [agent.env] is the agent's, and - # [verifier.env] is applied where the verifier runs — none silently lost. - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n' - '[environment.env]\nSHARED = "both"\n\n' - '[agent.env]\nAGENT_ONLY = "yes"\n\n' - '[verifier.env]\nVERIFIER_ONLY = "yes"\n', - encoding="utf-8", - ) +async def test_multi_step_tasks_are_refused_directly(tmp_path: Path, fake_docker) -> None: + make_multi_step_task(tmp_path, "multi") - parsed = harbor_load.HarborTask.read(task) - config = parsed.config + with pytest.raises(NotImplementedError, match="multi-step"): + await harbor.adapt(tmp_path) - # Container-wide reaches every process; the agent's reaches its sessions - # only; the verifier's is applied where the verifier runs. - assert config.environment.env == {"SHARED": "both"} - assert config.agent.env == {"AGENT_ONLY": "yes"} - assert config.verifier.env == {"VERIFIER_ONLY": "yes"} + assert fake_docker == [] -def test_only_a_real_user_conflict_is_refused(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[agent]\nuser = "app"\n', - encoding="utf-8", - ) +async def test_invalid_task_config_is_not_silently_defaulted( + tmp_path: Path, + fake_docker, +) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text("[environment]\ncpus = 'many'\n", encoding="utf-8") - # One phase naming an identity is fine: the image's single USER is it. - parsed = harbor_load.HarborTask.read(task) - assert parsed.unsupported_features() == [] - assert parsed.config.agent.user == "app" + with pytest.raises(ValueError, match="not a valid Harbor task"): + await harbor.adapt(tmp_path) + assert fake_docker == [] -def test_an_explicit_zero_timeout_is_not_silently_extended(tmp_path) -> None: - task = _write_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - 'schema_version = "1.3"\n\n[task]\nname = "demo/task-a"\n\n[verifier]\ntimeout_sec = 0\n', - encoding="utf-8", - ) - with pytest.raises(ValueError, match="not a valid Harbor task"): - harbor.load(tmp_path) - - -async def test_a_cancelled_grade_leaves_nothing_running(tmp_path) -> None: - # However grading exits — including a cancelled rollout — the verifier's - # process group and the pipe readers are released. - import asyncio - - from hud.utils.process import create_process_group_exec - from integrations.harbor._runtime import _grade_with_verifier - - task = _write_harbor_task(tmp_path, "task-a") - logs = tmp_path / "logs" - (logs / "verifier").mkdir(parents=True) - started: list[int] = [] - - async def run_tests(): - group = await create_process_group_exec( - "bash", - "-c", - "sleep 30", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - started.append(group.process.pid) - return group - - grading = asyncio.create_task( - _grade_with_verifier(harbor_load.TaskConfig.read(task), logs, None, run_tests) - ) - await asyncio.sleep(0.2) - grading.cancel() - with pytest.raises(asyncio.CancelledError): - await grading +async def test_task_symlinks_are_copied_without_reading_host_files( + tmp_path: Path, + fake_docker, +) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("host secret", encoding="utf-8") + task = make_harbor_task(tmp_path / "dataset", "task-a") + (task / "tests" / "link").symlink_to(outside) - # The verifier process is gone rather than orphaned for 30 seconds. - await asyncio.sleep(0.2) - with pytest.raises(ProcessLookupError): - os.kill(started[0], 0) + await harbor.adapt(task.parent) + (context,) = (task.parent / ".hud-adapt").iterdir() + copied = context / "tasks" / "task-a" / "tests" / "link" + assert copied.is_symlink() + assert os.readlink(copied) == str(outside) -def test_image_tags_do_not_depend_on_host_state(tmp_path) -> None: - # Links are copied as links, so hashing must read the link — not what it - # points at — or the same dataset tags differently on another machine. - from integrations.harbor._load import hash_directory - target = tmp_path / "outside.txt" - target.write_text("first", encoding="utf-8") - context = tmp_path / "ctx" - context.mkdir() - (context / "link").symlink_to(target) +async def test_adapt_hashes_links_not_their_targets( + tmp_path: Path, + fake_docker, +) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("first", encoding="utf-8") + task = make_harbor_task(tmp_path / "dataset", "task-a") + (task / "environment" / "link").symlink_to(outside) - before = hash_directory(context) - target.write_text("second, changed on this host only", encoding="utf-8") + (before,) = list(await harbor.adapt(task.parent)) + outside.write_text("changed", encoding="utf-8") + (after,) = list(await harbor.adapt(task.parent)) - assert hash_directory(context) == before + assert before.runtime_config == after.runtime_config -def test_a_workdir_inside_the_reserved_path_is_refused(tmp_path) -> None: - # That tree belongs to the adaptation layer and is hidden from sessions; - # a task working there would find it empty. - task = _write_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - f"FROM python:3.12-slim\nWORKDIR {harbor_load.HUD_ROOT}/app\n", encoding="utf-8" +def test_authored_runtime_assets_are_valid_source() -> None: + integration = Path(__file__).parents[1] + compile((integration / "env.py").read_text("utf-8"), "env.py", "exec") + result = subprocess.run( + ["sh", "-n", integration / "install.sh"], + check=False, + capture_output=True, ) + assert result.returncode == 0, result.stderr.decode() + - assert "reserved" in " ".join(harbor_load.HarborTask.read(task).unsupported_features()) +def test_public_surface_is_only_the_two_real_operations() -> None: + assert harbor.__all__ == ["adapt", "export"] diff --git a/integrations/harbor/tests/test_harbor.py b/integrations/harbor/tests/test_harbor.py index 8a6af370d..caa032460 100644 --- a/integrations/harbor/tests/test_harbor.py +++ b/integrations/harbor/tests/test_harbor.py @@ -3,141 +3,17 @@ from __future__ import annotations import asyncio +import json import textwrap from typing import TYPE_CHECKING import pytest -from integrations.harbor import detect, export, load - -from .conftest import make_harbor_task, make_multi_step_task +from integrations.harbor import export if TYPE_CHECKING: from pathlib import Path -# ─── detect / load: Harbor dirs -> Taskset ───────────────────────────── - - -def test_detect_recognizes_task_and_dataset_dirs(single_task: Path, tmp_path: Path) -> None: - assert detect(single_task) - assert detect(single_task.parent) # dataset dir containing task dirs - empty = tmp_path / "empty" - empty.mkdir() - assert not detect(empty) - assert not detect(single_task / "task.toml") # a file is not a task dir - - -def test_load_single_task_dir_maps_rows(single_task: Path) -> None: - taskset = load(single_task) - - assert len(taskset) == 1 - row = taskset["cancel-async-tasks"] - assert row.id == "cancel-async-tasks" - assert row.args == {} - assert row.env.startswith(f"{taskset.name}-") - - -def test_load_dataset_shares_one_env_per_build_context(dataset_same_env: Path) -> None: - taskset = load(dataset_same_env) - - assert len(taskset) == 3 - # Identical Dockerfiles -> all rows reference one env name. - (env_name,) = taskset.environment_names() - assert env_name.startswith("terminal-bench-sample-") - - -def test_load_dataset_groups_by_distinct_build_contexts(dataset_multi_env: Path) -> None: - taskset = load(dataset_multi_env) - - assert len(taskset) == 4 - names = taskset.environment_names() - assert len(names) == 2 - assert all(name.startswith("mixed-bench-") for name in names) - assert taskset["build-pmars"].env == taskset["cancel-async-tasks"].env - assert taskset["caffe-cifar-10"].env == taskset["sam-cell-seg"].env - assert taskset["build-pmars"].env != taskset["caffe-cifar-10"].env - - -def test_env_name_survives_other_tasks_joining_the_dataset(tmp_path: Path) -> None: - """A row's env name follows its own environment, not the dataset's shape. - - Names used to be positional (``-g1``, ``-g2``, assigned largest group - first), so growing one group renumbered the others and rows kept naming - an environment that had come to mean a different image. - """ - dataset = tmp_path / "bench" - dataset.mkdir() - solo_image, pair_image = "FROM alpine:3\n", "FROM debian:12\n" - make_harbor_task(dataset, "solo", dockerfile=solo_image) - for name in ("pair-a", "pair-b"): - make_harbor_task(dataset, name, dockerfile=pair_image) - - before = load(dataset) - solo_env = before["solo"].env - pair_env = before["pair-a"].env - assert solo_env != pair_env - - # The smaller group overtakes the larger one; under positional naming the - # two names swapped, silently re-pointing every row in both. - for name in ("solo-b", "solo-c"): - make_harbor_task(dataset, name, dockerfile=solo_image) - - after = load(dataset) - assert after["solo"].env == solo_env - assert after["pair-a"].env == pair_env - assert after["solo-b"].env == solo_env - - -def test_load_rejects_dirs_without_harbor_tasks(tmp_path: Path) -> None: - empty = tmp_path / "empty" - empty.mkdir() - with pytest.raises(ValueError, match="no Harbor tasks"): - load(empty) - - -def test_detect_and_load_recognize_multi_step_tasks(tmp_path: Path) -> None: - # A multi-step task has no root instruction.md; its instructions live under - # steps//, declared by a [[steps]] array in task.toml. - task = make_multi_step_task(tmp_path, "multi") - - assert detect(task) - assert {row.id for row in load(task)} == {"multi"} - - -def test_load_keeps_multi_step_tasks_alongside_single_step(tmp_path: Path) -> None: - dataset = tmp_path / "bench" - dataset.mkdir() - make_harbor_task(dataset, "single") - make_multi_step_task(dataset, "multi") - - assert {row.id for row in load(dataset)} == {"single", "multi"} - - -def test_detect_rejects_task_toml_without_instruction_or_steps(tmp_path: Path) -> None: - # task.toml alone is not a task: it needs a root instruction.md (single-step) - # or a [[steps]] array (multi-step). - task = tmp_path / "bare" - task.mkdir() - (task / "task.toml").write_text('[metadata]\ncategory = "x"\n', encoding="utf-8") - - assert not detect(task) - with pytest.raises(ValueError, match="no Harbor tasks"): - load(task) - - -def test_load_skips_unparseable_toml_but_keeps_the_rest(tmp_path: Path) -> None: - dataset = tmp_path / "bench" - dataset.mkdir() - make_harbor_task(dataset, "good") - broken = make_harbor_task(dataset, "broken") - (broken / "task.toml").write_text("not [valid toml", encoding="utf-8") - - taskset = load(dataset) - - # Unparseable config degrades gracefully; the task itself still loads. - assert {task.id for task in taskset} == {"good", "broken"} - - # ─── export: HUD tasks -> Harbor task folders ─────────────────────────── _ENV_PY = """\ @@ -259,12 +135,15 @@ async def test_environment_context_keeps_its_own_json_files(tmp_path: Path) -> N assert (created[0] / "environment" / "package.json").is_file() -async def test_dockerfile_neutralizes_cmd_and_bakes_boot(tmp_path: Path) -> None: +async def test_dockerfile_appends_the_harbor_runtime(tmp_path: Path) -> None: _write_env(tmp_path) created = await export(str(tmp_path / "env.py"), tmp_path / "out") dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert "# [hud original]" in dockerfile # original CMD neutralized + assert "COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh" in dockerfile assert 'ENTRYPOINT ["/hud_entrypoint.sh"]' in dockerfile + assert dockerfile.rindex('CMD ["sh", "-c", "sleep infinity"]') > dockerfile.index( + 'CMD ["hud", "serve", "env:env"]' + ) # The env build context is copied so the image can be rebuilt under Harbor. assert (created[0] / "environment" / "env.py").exists() @@ -300,9 +179,7 @@ async def test_export_survives_a_restrictive_dockerignore(tmp_path: Path) -> Non assert "!hud_entrypoint.sh" in ignore -async def test_export_neutralizes_multiline_startup_directives(tmp_path: Path) -> None: - # A backslash-continued CMD is one instruction: commenting its first line - # only would leave the rest as invalid top-level directives. +async def test_export_overrides_multiline_startup_directives(tmp_path: Path) -> None: _write_env(tmp_path) (tmp_path / "Dockerfile").write_text( 'FROM python:3.12-slim\nCMD ["python", \\\n "-m", \\\n "app"]\n', encoding="utf-8" @@ -311,15 +188,10 @@ async def test_export_neutralizes_multiline_startup_directives(tmp_path: Path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - active = [ - line - for line in dockerfile.splitlines() - if line.strip() and not line.strip().startswith("#") - ] - assert not any('"-m"' in line or '"app"' in line for line in active) + assert dockerfile.rindex('CMD ["sh", "-c", "sleep infinity"]') > dockerfile.index('"app"]') -async def test_export_restores_a_non_root_runtime_user(tmp_path: Path) -> None: +async def test_export_preserves_a_non_root_runtime_user(tmp_path: Path) -> None: _write_env(tmp_path) (tmp_path / "Dockerfile").write_text( "FROM python:3.12-slim\nRUN useradd -m app\nUSER app\n", encoding="utf-8" @@ -328,9 +200,8 @@ async def test_export_restores_a_non_root_runtime_user(tmp_path: Path) -> None: created = await export(str(tmp_path / "env.py"), tmp_path / "out") dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - boot = dockerfile[dockerfile.index("HUD → Harbor boot entrypoint") :] - assert "USER root" in boot # COPY writes root-owned; chmod needs it - assert boot.rindex("USER app") > boot.index("USER root") + assert dockerfile.count("USER ") == 1 + assert "COPY --chmod=0755" in dockerfile async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: @@ -351,12 +222,22 @@ async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None: - from integrations.harbor._export import _safe_component + _write_env(tmp_path) + taskset = tmp_path / "tasks.json" + taskset.write_text( + json.dumps([{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": "suite/fix"}]), + encoding="utf-8", + ) + + created = await export(str(taskset), tmp_path / "out") - assert "/" not in _safe_component("suite/fix") - assert _safe_component("../escape") == "escape" + assert created[0].name == "suite-fix" + taskset.write_text( + json.dumps([{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": ".."}]), + encoding="utf-8", + ) with pytest.raises(ValueError, match="usable directory name"): - _safe_component("..") + await export(str(taskset), tmp_path / "invalid") async def test_export_never_scores_a_broken_grader_as_zero(tmp_path: Path) -> None: From d3859969f7be2a80af4f78197e54a773f83b207c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 03:12:17 -0700 Subject: [PATCH 24/33] refactor(integrations): simplify Harbor conversion paths --- integrations/harbor/Dockerfile | 1 + integrations/harbor/adapt.py | 58 +++++++++++------------- integrations/harbor/env.py | 36 ++++++--------- integrations/harbor/export.py | 48 +++++++++----------- integrations/harbor/install.sh | 44 ++++-------------- integrations/harbor/tests/test_harbor.py | 8 ++-- 6 files changed, 75 insertions(+), 120 deletions(-) diff --git a/integrations/harbor/Dockerfile b/integrations/harbor/Dockerfile index 3118ca4f3..ebf770227 100644 --- a/integrations/harbor/Dockerfile +++ b/integrations/harbor/Dockerfile @@ -3,6 +3,7 @@ FROM ${BASE_IMAGE} USER root ARG HUD_REQUIREMENT=hud +COPY --from=ghcr.io/astral-sh/uv:0.8.15 /uv /media/hud/bin/uv COPY env.py install.sh tasks.json /media/hud/ COPY tasks /media/hud/tasks COPY packages /media/hud/packages diff --git a/integrations/harbor/adapt.py b/integrations/harbor/adapt.py index b8e1957b4..1d3a6bad6 100644 --- a/integrations/harbor/adapt.py +++ b/integrations/harbor/adapt.py @@ -92,6 +92,7 @@ class HarborTask: path: Path config: TaskConfig environment_hash: str + runtime: dict[str, Any] async def adapt( @@ -156,43 +157,41 @@ async def adapt( + ", ".join(unsupported) ) + environment = config.environment tasks.append( HarborTask( path=task_dir, config=config, environment_hash=_tree_hash(task_dir / "environment"), + runtime={ + "image": environment.docker_image, + "workdir": environment.workdir, + "environment_env": environment.env, + "environment_network": environment.network_mode, + "environment_hosts": environment.allowed_hosts, + "agent": config.agent.model_dump( + include={"user", "network_mode", "allowed_hosts", "env"} + ), + "verifier": config.verifier.model_dump( + include={"user", "network_mode", "allowed_hosts", "env"} + ), + }, ) ) grouped: dict[tuple[str, str], list[HarborTask]] = {} for task in tasks: - config = task.config - runtime = json.dumps( - { - "image": config.environment.docker_image, - "workdir": config.environment.workdir, - "environment_env": config.environment.env, - "environment_network": config.environment.network_mode, - "environment_hosts": config.environment.allowed_hosts, - "agent": config.agent.model_dump( - include={"user", "network_mode", "allowed_hosts", "env"} - ), - "verifier": config.verifier.model_dump( - include={"user", "network_mode", "allowed_hosts", "env"} - ), - }, - sort_keys=True, - ) - grouped.setdefault((task.environment_hash, runtime), []).append(task) + runtime_json = json.dumps(task.runtime, sort_keys=True) + grouped.setdefault((task.environment_hash, runtime_json), []).append(task) rows = [] base_name = normalize_environment_name(dataset.name, default="harbor") - for (environment_hash, runtime), group in sorted(grouped.items()): - digest = hashlib.sha256((environment_hash + "\0" + runtime).encode()).hexdigest()[:12] + for (environment_hash, runtime_json), group in sorted(grouped.items()): + digest = hashlib.sha256((environment_hash + "\0" + runtime_json).encode()).hexdigest()[:12] name = f"{base_name}-{digest}" source = group[0] dockerfile = source.path / "environment" / "Dockerfile" - base_image = source.config.environment.docker_image + base_image = source.runtime["image"] if base_image and not dockerfile.is_file(): await docker("pull", base_image) elif dockerfile.is_file(): @@ -223,8 +222,7 @@ async def adapt( for asset in ("Dockerfile", "env.py", "install.sh"): shutil.copy2(ASSETS / asset, context / asset) - environment = source.config.environment - workdir = environment.workdir or image_config.get("WorkingDir") or "/" + workdir = source.runtime["workdir"] or image_config.get("WorkingDir") or "/" if Path(workdir).is_relative_to(HUD_ROOT): raise ValueError(f"Harbor workdir {workdir!r} is inside reserved path {HUD_ROOT}") manifest = { @@ -232,16 +230,12 @@ async def adapt( "workdir": workdir, "image_user": image_config.get("User") or None, "environment": { - "env": environment.env, - "network_mode": environment.network_mode, - "allowed_hosts": environment.allowed_hosts, + "env": source.runtime["environment_env"], + "network_mode": source.runtime["environment_network"], + "allowed_hosts": source.runtime["environment_hosts"], }, - "agent": source.config.agent.model_dump( - include={"user", "network_mode", "allowed_hosts", "env"} - ), - "verifier": source.config.verifier.model_dump( - include={"user", "network_mode", "allowed_hosts", "env"} - ), + "agent": source.runtime["agent"], + "verifier": source.runtime["verifier"], "tasks": [], } for task in group: diff --git a/integrations/harbor/env.py b/integrations/harbor/env.py index e6df4703c..150f3dbd6 100644 --- a/integrations/harbor/env.py +++ b/integrations/harbor/env.py @@ -63,24 +63,19 @@ def home(user_id: int | None) -> str | None: return None -def harness_mounts() -> tuple[Mount, ...]: - parent = ROOT.parent - siblings = ( - [ - Mount("rw", src=str(path), dst=str(path)) - for path in sorted(parent.iterdir()) - if path != ROOT - ] - if parent.is_dir() - else [] - ) - return (Mount("tmpfs", dst=str(parent)), *siblings) - - agent = CONFIG["agent"] agent_uid = uid(agent) agent_network, agent_hosts = network(agent) rooted_at_filesystem = len(WORKDIR.parts) == 1 +harness_parent = ROOT.parent +harness_mounts = ( + Mount("tmpfs", dst=str(harness_parent)), + *( + Mount("rw", src=str(path), dst=str(path)) + for path in sorted(harness_parent.iterdir()) + if path != ROOT + ), +) env = Environment(CONFIG["name"]) workspace = env.workspace( @@ -91,7 +86,7 @@ def harness_mounts() -> tuple[Mount, ...]: Mount("proc", dst="/proc"), Mount("dev", dst="/dev"), ), - mounts=harness_mounts(), + mounts=harness_mounts, credentials_dir=ROOT / "session-keys", shell_uid=agent_uid, hand_over_root=False, @@ -154,7 +149,10 @@ async def grade(task_dir: Path, timeout_sec: float, answer: Any) -> EvaluationRe test_script.chmod(test_script.stat().st_mode | 0o111) clear(VERIFIER_LOGS) - write(LOGS / "agent_answer.txt", "" if answer is None else str(answer)) + answer_file = LOGS / "agent_answer.txt" + if answer_file.is_symlink(): + answer_file.unlink() + answer_file.write_text("" if answer is None else str(answer), encoding="utf-8") verifier = CONFIG["verifier"] verifier_uid = uid(verifier) @@ -299,9 +297,3 @@ def reward() -> tuple[float | None, dict[str, Any]]: return score, {"reward_file": str(reward_text)} return None, {"reward_parse_error": text} return None, {} - - -def write(path: Path, text: str) -> None: - if path.is_symlink(): - path.unlink() - path.write_text(text, encoding="utf-8") diff --git a/integrations/harbor/export.py b/integrations/harbor/export.py index 9f175f2d7..5811d91a7 100644 --- a/integrations/harbor/export.py +++ b/integrations/harbor/export.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any -from hud.environment import Environment +from hud.environment import Environment, load_environment from hud.environment.server import TaskRunner from hud.eval import Taskset from hud.utils.naming import normalize_environment_name @@ -64,8 +64,6 @@ async def export( authored environment and Dockerfile. Each task becomes one self-contained Harbor task folder. """ - from hud.utils.modules import iter_modules - src = Path(source).resolve() source_dir = src.parent if src.is_file() else src out = Path(out_dir).resolve() @@ -73,14 +71,11 @@ async def export( tasks = list(Taskset.from_file(src)) scan = source_dir if src.suffix in (".json", ".jsonl") else src - authored: dict[str, Environment] = {} - serve_targets: dict[str, str] = {} - for module in iter_modules(scan): - module_file = Path(getattr(module, "__file__", "") or "") - for attribute, value in vars(module).items(): - if isinstance(value, Environment): - authored[value.name] = value - serve_targets[value.name] = f"{module_file.stem or 'env'}:{attribute}" + authored = { + name: load_environment(scan, name=name) + for name in dict.fromkeys(task.env for task in tasks) + } + serve_source = src.name if src.suffix == ".py" else "." dockerfile = next( ( @@ -106,19 +101,11 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: created: list[Path] = [] claimed: dict[str, str] = {} - started: set[str] = set() + started: list[Environment] = [] try: - for task in tasks: - env = authored.get(task.env) - if env is None or task.id not in env.tasks: - raise TypeError( - f"harbor export needs a local env defining task {task.id!r} " - f"(an env.py named {task.env!r} next to the tasks); none was found.", - ) - if env.name not in started: - started.add(env.name) - await env.start() - + for env in authored.values(): + started.append(env) + await env.start() unsupported = [ capability.protocol for capability in env.capabilities @@ -130,6 +117,14 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: f"only {'/'.join(ALLOWED_PROTOCOLS)} are convertible.", ) + for task in tasks: + env = authored[task.env] + if task.id not in env.tasks: + raise TypeError( + f"harbor export needs a local env defining task {task.id!r} " + f"(an env.py named {task.env!r} next to the tasks); none was found.", + ) + declared = task.slug if not any(character.isalnum() for character in declared): raise ValueError(f"task slug {declared!r} does not form a usable directory name") @@ -199,11 +194,10 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: if alternate.exists(): alternate.unlink() - serve_target = serve_targets[env.name] (env_out / "hud_entrypoint.sh").write_text( ENTRYPOINT_SH.format( port=CONTROL_PORT, - serve_target=shlex.quote(serve_target), + serve_target=shlex.quote(f"{serve_source}:{env.name}"), task=shlex.quote(task.id), args_json=shlex.quote(args_json), ), @@ -232,7 +226,7 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: ) created.append(task_dir) finally: - for name in started: - await authored[name].stop() + for env in reversed(started): + await env.stop() return created diff --git a/integrations/harbor/install.sh b/integrations/harbor/install.sh index a24cdfc55..f8c265370 100644 --- a/integrations/harbor/install.sh +++ b/integrations/harbor/install.sh @@ -5,49 +5,23 @@ requirement="${1:-hud}" root=/media/hud python_version=3.12 -export UV_INSTALL_DIR="$root/bin" export UV_PYTHON_INSTALL_DIR="$root/python" export UV_PYTHON_BIN_DIR="$root/bin" export UV_NO_CACHE=1 export XDG_CONFIG_HOME="$root/config" export PATH="$root/bin:$PATH" -if ! command -v uv >/dev/null 2>&1; then - if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then - { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \ - || apk add --no-cache curl ca-certificates - fi - { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \ - || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \ - || pip install -q -U uv +if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq bubblewrap python3 python3-venv python3-pip git curl ca-certificates + rm -rf /var/lib/apt/lists/* +elif command -v apk >/dev/null 2>&1; then + apk add --no-cache bubblewrap python3 py3-pip git curl ca-certificates +else + echo "hud: Harbor environments require an apt- or apk-based image" >&2 + exit 1 fi -command -v bwrap >/dev/null 2>&1 \ - || { apt-get update -qq && apt-get install -y -qq bubblewrap; } \ - || apk add --no-cache bubblewrap - uv python install "$python_version" uv venv "$root/venv" --python "$python_version" uv pip install --python "$root/venv/bin/python" "$requirement" - -apt_packages="" -apk_packages="" -for spec in \ - "python3|python3 python3-venv|python3" \ - "pip3|python3-pip|py3-pip" \ - "git|git|git" \ - "curl|curl ca-certificates|curl ca-certificates" -do - command="${spec%%|*}" - if command -v "$command" >/dev/null 2>&1; then - continue - fi - rest="${spec#*|}" - apt_packages="$apt_packages ${rest%%|*}" - apk_packages="$apk_packages ${rest##*|}" -done - -if [ -n "$apt_packages" ]; then - { apt-get update -qq && apt-get install -y -qq $apt_packages; } \ - || apk add --no-cache $apk_packages -fi diff --git a/integrations/harbor/tests/test_harbor.py b/integrations/harbor/tests/test_harbor.py index caa032460..d06886d50 100644 --- a/integrations/harbor/tests/test_harbor.py +++ b/integrations/harbor/tests/test_harbor.py @@ -100,7 +100,7 @@ async def test_scripts_drive_hud_task_lifecycle(tmp_path: Path) -> None: test_sh = (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") # Boot serves the channel, parks the run via setup, then hands off. - assert "hud serve env:env" in boot + assert "hud serve env.py:demo" in boot assert "hud task start solve" in boot assert "python3 -c" not in boot assert 'exec "$@"' in boot @@ -205,8 +205,8 @@ async def test_export_preserves_a_non_root_runtime_user(tmp_path: Path) -> None: async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: - # The env lives in tasks.py under the name ``bench`` — the container must - # serve that, not a guessed ``env:env``. + # The environment is bound to ``bench`` in tasks.py, while its public name + # remains ``demo``. The generated target uses the canonical loader syntax. (tmp_path / "tasks.py").write_text( textwrap.dedent(_ENV_PY) .replace("env = Environment", "bench = Environment") @@ -218,7 +218,7 @@ async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: created = await export(str(tmp_path / "tasks.py"), tmp_path / "out") boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - assert "hud serve tasks:bench" in boot + assert "hud serve tasks.py:demo" in boot async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None: From 6b6efcadba7183f978e63790984d13d62b29b870 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:30:29 -0700 Subject: [PATCH 25/33] refactor(environment): consolidate workspace execution --- hud/environment/tests/test_workspace.py | 76 ++++++++++++----- hud/environment/workspace.py | 97 +++++++++++++++++++++- integrations/harbor/adapt.py | 4 + integrations/harbor/env.py | 92 +++----------------- integrations/harbor/tests/test_contract.py | 5 ++ 5 files changed, 170 insertions(+), 104 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index d603715e5..05487d041 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import itertools import os import shutil @@ -12,6 +13,7 @@ from pathlib import Path from types import SimpleNamespace from typing import Any, cast +from unittest.mock import AsyncMock import asyncssh import pytest @@ -19,6 +21,7 @@ from hud.capabilities import SSHClient from hud.environment import workspace as workspace_mod from hud.environment.workspace import Mount, Workspace +from hud.utils.process import ProcessResult pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX workspace semantics") @@ -390,34 +393,65 @@ def test_making_a_network_and_joining_it_are_the_same_question( assert ("--net" in ws.enter_argv(7, "true")) is owns -def test_process_builders_can_apply_a_phase_policy( +@pytest.mark.asyncio +async def test_run_uses_the_shared_sandbox_and_visitor_egress( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - ws = Workspace(tmp_path / "root", network=True, env={"AGENT_ONLY": "yes"}) - monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + ws = Workspace(tmp_path / "root") + + @contextlib.asynccontextmanager + async def visiting(allowed): + assert allowed == {"pypi.org"} + yield {"HTTPS_PROXY": "http://visitor"} + + complete = AsyncMock(return_value=ProcessResult(0, b"passed", b"")) + spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) - verifier = ws.bwrap_argv( - ["true"], - env={"VERIFIER_ONLY": "yes"}, - inherit_workspace_env=False, - network=False, - mount_hosts=False, + monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) + monkeypatch.setattr(ws, "visiting", visiting) + monkeypatch.setattr(workspace_mod, "create_process_group_exec", spawn) + + result = await ws.run( + ["test.sh"], + identity=None, + allowed_hosts={"pypi.org"}, + max_wait=12, ) - entered = ws.enter_argv( - 7, - ["true"], - env={"VERIFIER_ONLY": "yes"}, + + assert result.stdout == b"passed" + complete.assert_awaited_once_with(max_wait=12) + argv, kwargs = spawn.await_args + assert argv[argv.index("--target") + 1] == "7" + assert "HTTPS_PROXY=http://visitor" in argv + assert kwargs["env"]["HTTPS_PROXY"] == "http://visitor" + + +@pytest.mark.asyncio +async def test_run_can_use_a_fresh_no_network_sandbox( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + ws = Workspace(tmp_path / "root") + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + install_identity_map = AsyncMock(return_value=9) + complete = AsyncMock(return_value=ProcessResult(0, b"isolated", b"")) + spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) + + monkeypatch.setattr(workspace_mod, "install_identity_map", install_identity_map) + monkeypatch.setattr(workspace_mod, "create_process_group_exec", spawn) + + result = await ws.run( + ["test.sh"], + isolated=True, identity=None, - inherit_workspace_env=False, - preserve_credentials=True, + max_wait=5, ) - assert "--unshare-net" in verifier - assert "AGENT_ONLY=yes" not in verifier - assert "VERIFIER_ONLY=yes" in verifier - assert "--preserve-credentials" in entered - assert "AGENT_ONLY=yes" not in entered - assert "VERIFIER_ONLY=yes" in entered + assert result.stdout == b"isolated" + complete.assert_awaited_once_with(max_wait=5) + install_identity_map.assert_awaited_once() + argv, kwargs = spawn.await_args + assert "--unshare-net" in argv + assert len(kwargs["pass_fds"]) == 2 def test_a_peer_answers_at_the_address_the_task_expects() -> None: diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index d6440ddfc..f3616a4ba 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -20,7 +20,7 @@ import asyncssh from hud.environment.egress import VISITOR_PORT, Egress, Peer, hosts_text, proxy_environment -from hud.utils.process import create_process_group_exec +from hud.utils.process import ProcessResult, create_process_group_exec if sys.platform != "win32": # the pty a session runs on has no Windows analogue import fcntl @@ -696,6 +696,101 @@ def file_tracking_capability(self, name: str = "filetracking") -> Capability: params={"root": self.root.as_posix(), "setup_diff": True}, ) + async def run( + self, + command: list[str], + *, + isolated: bool = False, + env: Mapping[str, str] | None = None, + identity: int | None | Literal["workspace"] = "workspace", + inherit_workspace_env: bool = True, + allowed_hosts: Collection[str] = (), + no_new_privs: bool = True, + max_wait: float | None = None, + ) -> ProcessResult: + """Run a captured command against this workspace. + + Normally the command joins the persistent sandbox used by SSH sessions, + so it can inspect processes the agent started. ``allowed_hosts`` opens a + separate egress policy only for the command's lifetime. ``isolated=True`` + instead creates a fresh no-network sandbox over the same filesystem. + """ + if ( + isinstance(identity, int) + and hasattr(os, "geteuid") + and identity != os.geteuid() + and not (_is_root() and self._setpriv() is not None) + ): + raise RuntimeError("setpriv is required to run a workspace command as another user") + + process_env = dict(env or {}) + if not isolated: + sandbox = await self.sandbox_pid() + if sandbox is None: + raise RuntimeError("workspace commands require a live sandbox") + async with self.visiting(allowed_hosts) as visitor_env: + process_env.update(visitor_env) + process = await create_process_group_exec( + *self.enter_argv( + sandbox, + command, + env=process_env, + identity=identity, + inherit_workspace_env=inherit_workspace_env, + preserve_credentials=True, + no_new_privs=no_new_privs, + ), + cwd=self.root, + env={**os.environ, **process_env}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + return await process.complete(max_wait=max_wait) + + if allowed_hosts: + raise ValueError("an isolated workspace command has no network") + + info_read, info_write = os.pipe() + block_read, block_write = os.pipe() + try: + os.set_inheritable(info_write, True) + os.set_inheritable(block_read, True) + if identity == "workspace": + drop = self._drop_argv(no_new_privs=no_new_privs) + elif identity is None: + drop = [] + else: + drop = self._drop_argv(identity, no_new_privs=no_new_privs) + process = await create_process_group_exec( + *self.bwrap_argv( + [*drop, *command], + env=process_env, + inherit_host_env=False, + inherit_workspace_env=inherit_workspace_env, + info_fd=info_write, + userns_block_fd=block_read, + network=False, + mount_hosts=False, + isolate_processes=False, + ), + cwd=self.root, + env={**os.environ, **process_env}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + pass_fds=(info_write, block_read), + ) + os.close(info_write) + os.close(block_read) + info_write = block_read = -1 + await install_identity_map(info_read, block_write) + finally: + os.close(info_read) + os.close(block_write) + for descriptor in (info_write, block_read): + if descriptor != -1: + os.close(descriptor) + return await process.complete(max_wait=max_wait) + # ─── argv builders (public — useful if you want your own subprocess) ── @property diff --git a/integrations/harbor/adapt.py b/integrations/harbor/adapt.py index 1d3a6bad6..98c984a1a 100644 --- a/integrations/harbor/adapt.py +++ b/integrations/harbor/adapt.py @@ -131,6 +131,10 @@ async def adapt( unsupported.append(f"os={config.environment.os!r}") if config.environment.tpu: unsupported.append("TPUs") + if len(config.environment.gpu_types) > 1: + unsupported.append("multiple GPU types") + elif config.environment.gpu_types and not config.environment.gpus: + unsupported.append("GPU types without GPUs") if config.environment.healthcheck: unsupported.append("healthcheck") if config.environment.mcp_servers: diff --git a/integrations/harbor/env.py b/integrations/harbor/env.py index 150f3dbd6..2bd2cfa11 100644 --- a/integrations/harbor/env.py +++ b/integrations/harbor/env.py @@ -2,7 +2,6 @@ from __future__ import annotations -import asyncio import contextlib import json import math @@ -15,9 +14,7 @@ from hud.environment import Environment, Mount from hud.environment.egress import ANY_HOST -from hud.environment.workspace import install_identity_map from hud.graders import EvaluationResult -from hud.utils.process import ProcessGroup, create_process_group_exec ROOT = Path("/media/hud") TESTS = Path("/tests") @@ -158,8 +155,6 @@ async def grade(task_dir: Path, timeout_sec: float, answer: Any) -> EvaluationRe verifier_uid = uid(verifier) verifier_env = dict(verifier["env"]) if verifier_uid is not None: - if os.geteuid() == 0 and shutil.which("setpriv") is None: - raise RuntimeError("setpriv is required to run the Harbor verifier as another user") for root in (TESTS, VERIFIER_LOGS): for path in (root, *root.rglob("*")): os.lchown(path, verifier_uid, verifier_uid) @@ -167,31 +162,16 @@ async def grade(task_dir: Path, timeout_sec: float, answer: Any) -> EvaluationRe verifier_env["HOME"] = verifier_home verifier_network, verifier_hosts = network(verifier) - command = [str(test_script)] - if verifier_network: - sandbox = await workspace.sandbox_pid() - if sandbox is None: - raise RuntimeError("the Harbor verifier requires the workspace sandbox") - async with workspace.visiting(verifier_hosts) as visitor_env: - process = await create_process_group_exec( - *workspace.enter_argv( - sandbox, - command, - env={**verifier_env, **visitor_env}, - identity=verifier_uid, - inherit_workspace_env=False, - preserve_credentials=True, - no_new_privs=False, - ), - cwd=WORKDIR, - env={**os.environ, **verifier_env, **visitor_env}, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - execution = await process.complete(max_wait=timeout_sec) - else: - process = await isolated(command, verifier_env, verifier_uid) - execution = await process.complete(max_wait=timeout_sec) + execution = await workspace.run( + [str(test_script)], + isolated=not verifier_network, + env=verifier_env, + identity=verifier_uid, + inherit_workspace_env=False, + allowed_hosts=verifier_hosts, + no_new_privs=False, + max_wait=timeout_sec, + ) info: dict[str, Any] = { "exit_code": execution.returncode, @@ -217,58 +197,6 @@ async def grade(task_dir: Path, timeout_sec: float, answer: Any) -> EvaluationRe return EvaluationResult(reward=score, info=info) -async def isolated( - command: list[str], - verifier_env: dict[str, str], - verifier_uid: int | None, -) -> ProcessGroup: - info_read, info_write = os.pipe() - block_read, block_write = os.pipe() - try: - os.set_inheritable(info_write, True) - os.set_inheritable(block_read, True) - if verifier_uid is not None: - command = [ - shutil.which("setpriv") or "setpriv", - "--reuid", - str(verifier_uid), - "--regid", - str(verifier_uid), - "--clear-groups", - "--", - *command, - ] - process = await create_process_group_exec( - *workspace.bwrap_argv( - command, - cwd=WORKDIR.as_posix(), - env=verifier_env, - inherit_workspace_env=False, - info_fd=info_write, - userns_block_fd=block_read, - network=False, - mount_hosts=False, - isolate_processes=False, - ), - cwd=WORKDIR, - env={**os.environ, **verifier_env}, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - pass_fds=(info_write, block_read), - ) - os.close(info_write) - os.close(block_read) - info_write = block_read = -1 - await install_identity_map(info_read, block_write) - return process - finally: - os.close(info_read) - os.close(block_write) - for descriptor in (info_write, block_read): - if descriptor != -1: - os.close(descriptor) - - def reward() -> tuple[float | None, dict[str, Any]]: reward_json = VERIFIER_LOGS / "reward.json" if reward_json.is_file(): diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index 84effe924..b2c4242d0 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -201,6 +201,11 @@ async def test_runtime_configuration_is_data_not_dockerfile_codegen( [ ('[environment]\nos = "windows"\n', "os="), ('[environment]\ntpu = {type = "v5", topology = "2x2"}\n', "TPUs"), + ( + '[environment]\ngpus = 1\ngpu_types = ["H100", "A100"]\n', + "multiple GPU types", + ), + ('[environment]\ngpu_types = ["H100"]\n', "GPU types without GPUs"), ('[environment.healthcheck]\ncommand = "curl localhost"\n', "healthcheck"), ('[[environment.mcp_servers]]\nname = "db"\n', "MCP servers"), ('[verifier]\nenvironment_mode = "separate"\n', "separate verifier"), From dd46e602d72fd8aafaed9070ff5c5f5f5ebfcf4b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:37:55 -0700 Subject: [PATCH 26/33] fix(process): bound completion on leader exit --- hud/environment/tests/test_workspace.py | 2 + hud/utils/process.py | 54 +++++++++++------- hud/utils/tests/test_process.py | 73 +++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 21 deletions(-) create mode 100644 hud/utils/tests/test_process.py diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 05487d041..33e3b9453 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -420,6 +420,7 @@ async def visiting(allowed): assert result.stdout == b"passed" complete.assert_awaited_once_with(max_wait=12) + assert spawn.await_args is not None argv, kwargs = spawn.await_args assert argv[argv.index("--target") + 1] == "7" assert "HTTPS_PROXY=http://visitor" in argv @@ -449,6 +450,7 @@ async def test_run_can_use_a_fresh_no_network_sandbox( assert result.stdout == b"isolated" complete.assert_awaited_once_with(max_wait=5) install_identity_map.assert_awaited_once() + assert spawn.await_args is not None argv, kwargs = spawn.await_args assert "--unshare-net" in argv assert len(kwargs["pass_fds"]) == 2 diff --git a/hud/utils/process.py b/hud/utils/process.py index 9f800e845..fa67565bf 100644 --- a/hud/utils/process.py +++ b/hud/utils/process.py @@ -70,33 +70,45 @@ async def wait(self) -> int: wait_task.cancel() await asyncio.gather(wait_task, return_exceptions=True) - async def communicate( - self, - input: bytes | None = None, - *, - max_wait: float | None = None, - ) -> tuple[bytes, bytes]: - try: - if max_wait is None: - result = await self.process.communicate(input=input) - else: - result = await asyncio.wait_for(self.process.communicate(input=input), max_wait) - finally: - await self.terminate() - return result - async def complete( self, - input: bytes | None = None, *, max_wait: float | None = None, ) -> ProcessResult: - """Capture output and teardown, reporting timeout as process data.""" + """Capture output and teardown, reporting timeout as process data. + + The deadline follows the process leader rather than pipe EOF: a + background child may inherit the pipes after the leader has finished. + """ + stdout_read = ( + asyncio.create_task(self.process.stdout.read()) + if self.process.stdout is not None + else None + ) + stderr_read = ( + asyncio.create_task(self.process.stderr.read()) + if self.process.stderr is not None + else None + ) + readers = tuple(reader for reader in (stdout_read, stderr_read) if reader is not None) + timed_out = False try: - stdout, stderr = await self.communicate(input, max_wait=max_wait) - except TimeoutError: - return ProcessResult(self.returncode, b"", b"", timed_out=True) - return ProcessResult(self.returncode, stdout, stderr) + try: + await asyncio.wait_for(self.wait(), max_wait) + except TimeoutError: + timed_out = True + returncode = self.returncode + finally: + try: + await self.terminate() + finally: + await asyncio.gather(*readers) + return ProcessResult( + returncode, + stdout_read.result() if stdout_read is not None else b"", + stderr_read.result() if stderr_read is not None else b"", + timed_out, + ) async def terminate(self) -> None: await _terminate_process_group( diff --git a/hud/utils/tests/test_process.py b/hud/utils/tests/test_process.py new file mode 100644 index 000000000..5724fb7eb --- /dev/null +++ b/hud/utils/tests/test_process.py @@ -0,0 +1,73 @@ +"""What a managed process group reports when it ends in each of its ways.""" + +from __future__ import annotations + +import asyncio +import sys + +import pytest + +from hud.utils.process import ProcessResult, create_process_group_exec + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.skipif(sys.platform == "win32", reason="POSIX process groups"), +] + + +async def _run(script: str, max_wait: float | None = None) -> ProcessResult: + group = await create_process_group_exec( + "sh", + "-c", + script, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + return await group.complete(max_wait=max_wait) + + +async def test_a_finished_process_is_not_a_timeout_because_a_child_holds_its_pipes() -> None: + """Starting a service and exiting is how a verifier ends: it has already + written its verdict. Waiting on the inherited pipes instead of on the + process reports that as a timeout, and scores a passing task zero.""" + result = await _run("echo verdict-written; sleep 30 & exit 0", max_wait=10) + + assert result.timed_out is False + assert result.returncode == 0 + assert b"verdict-written" in result.stdout + + +async def test_a_process_that_overran_keeps_what_it_printed() -> None: + """The output is the evidence of how far it got; reporting only that the + deadline passed throws away the one thing that explains the timeout.""" + result = await _run("echo progress-so-far; sleep 30", max_wait=1) + + assert result.timed_out is True + assert b"progress-so-far" in result.stdout + + +async def test_a_chatty_process_does_not_block_on_a_full_pipe() -> None: + """Reading only after exit deadlocks once the pipe buffer fills (~64KB).""" + result = await asyncio.wait_for(_run("yes hud | head -c 500000; exit 0", max_wait=30), 30) + + assert result.timed_out is False + assert len(result.stdout) == 500000 + + +async def test_a_cancelled_call_leaves_nothing_running() -> None: + """A cancelled rollout unwinds through here, and the group is this call's + to release however it exits.""" + group = await create_process_group_exec( + "sh", + "-c", + "sleep 30", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + task = asyncio.create_task(group.complete()) + await asyncio.sleep(0.2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert group.returncode is not None From f4f19229986c17690494cfaad5e0121960559f8c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:21:18 -0700 Subject: [PATCH 27/33] fix(integrations): name the served environment as a literal in adapted contexts `hud deploy` resolves a context's identity from a literal Environment(...) name in source and refuses a computed one, so the copied env.py carries the group's name as a literal. The value is the same one tasks.json serves; a sentinel guard fails loudly if the asset's construction ever drifts. --- integrations/harbor/adapt.py | 14 +++++++++++++- integrations/harbor/tests/test_contract.py | 7 ++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/integrations/harbor/adapt.py b/integrations/harbor/adapt.py index 98c984a1a..0f128448f 100644 --- a/integrations/harbor/adapt.py +++ b/integrations/harbor/adapt.py @@ -223,8 +223,20 @@ async def adapt( shutil.rmtree(context) (context / "tasks").mkdir(parents=True) (context / "packages").mkdir() - for asset in ("Dockerfile", "env.py", "install.sh"): + for asset in ("Dockerfile", "install.sh"): shutil.copy2(ASSETS / asset, context / asset) + # ``hud deploy`` resolves the context's identity from a literal + # Environment(...) name in source, so the copy carries the group's + # name as a literal; the value is the same one tasks.json serves. + served = (ASSETS / "env.py").read_text("utf-8") + sentinel = 'Environment(CONFIG["name"])' + if sentinel not in served: + raise RuntimeError(f"env.py asset no longer constructs {sentinel}") + (context / "env.py").write_text( + served.replace(sentinel, f'Environment("{name}")'), + encoding="utf-8", + newline="\n", + ) workdir = source.runtime["workdir"] or image_config.get("WorkingDir") or "/" if Path(workdir).is_relative_to(HUD_ROOT): diff --git a/integrations/harbor/tests/test_contract.py b/integrations/harbor/tests/test_contract.py index b2c4242d0..03b9fed53 100644 --- a/integrations/harbor/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -50,8 +50,13 @@ async def test_adapt_builds_the_source_then_an_authored_hud_environment( (context,) = (tmp_path / ".hud-adapt").iterdir() integration = Path(__file__).parents[1] - for asset in ("Dockerfile", "env.py", "install.sh"): + for asset in ("Dockerfile", "install.sh"): assert (context / asset).read_bytes() == (integration / asset).read_bytes() + # env.py names the environment as a literal — `hud deploy` resolves the + # context's identity from source, and refuses a computed name. + served = (context / "env.py").read_text(encoding="utf-8") + assert f'Environment("{context.name}")' in served + assert 'Environment(CONFIG["name"])' not in served assert (context / "tasks" / "task-a" / "instruction.md").is_file() assert (context / "tasks" / "task-a" / "tests" / "test.sh").is_file() From 1806fb042c6f4d4fc7f8c4d0310d6abc8fec5d5c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:21:18 -0700 Subject: [PATCH 28/33] fix(environment): read bwrap's info document to completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single read of the info fd can return a prefix of the document — bwrap's write is not atomic with this side's read — and parsing the truncated JSON killed the sandbox, which took every SSH session and the grade down with it on a live box. Read until the document parses or the fd closes. A raising session factory is also logged now: asyncssh reports it to the client as a bare "Session request failed", so without a log line the reason existed nowhere. --- hud/environment/tests/test_workspace.py | 32 +++++++++++++++++++++++++ hud/environment/workspace.py | 20 ++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 33e3b9453..e92be76af 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -9,10 +9,12 @@ import shutil import sys import tempfile +import threading import time from pathlib import Path from types import SimpleNamespace from typing import Any, cast +from unittest import mock from unittest.mock import AsyncMock import asyncssh @@ -771,3 +773,33 @@ async def test_a_plain_root_publishes_no_alias(tmp_path: Path) -> None: assert "cwd_aliases" not in ws.capability().params finally: await ws.stop() + + +@pytest.mark.asyncio +async def test_identity_map_reads_a_chunked_info_document() -> None: + """bwrap's info JSON arrives in as many chunks as the pipe delivers; a + reader that stops at the first chunk parses a truncated document and the + sandbox never starts (observed as every session failing on a live box).""" + info_read, info_write = os.pipe() + block_read, block_write = os.pipe() + document = b'{\n "child-pid": 4242,\n "other": "field"\n}\n' + + def write_in_chunks() -> None: + os.write(info_write, document[:21]) # cut mid-document, line 2 + time.sleep(0.05) + os.write(info_write, document[21:]) + os.close(info_write) + + writer = threading.Thread(target=write_in_chunks) + writer.start() + try: + with mock.patch.object(workspace_mod, "_map_identities") as mapped: + pid = await workspace_mod.install_identity_map(info_read, block_write) + assert pid == 4242 + mapped.assert_called_once_with(4242) + assert os.read(block_read, 1) == b"\n" # the sandbox was released + finally: + writer.join() + for fd in (info_read, block_read, block_write): + with contextlib.suppress(OSError): + os.close(fd) diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index f3616a4ba..830c73ab8 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -198,7 +198,16 @@ async def install_identity_map(info_read: int, block_write: int) -> int: who its ids are, and only then does anything run in it. Returns that pid. """ loop = asyncio.get_running_loop() - raw = await asyncio.wait_for(loop.run_in_executor(None, os.read, info_read, 4096), 30.0) + # The info document arrives in as many chunks as the pipe delivers — a + # single read can return a prefix of it (bwrap's write is not atomic with + # this side's read). Read until the document parses or the fd closes. + raw = b"" + async with asyncio.timeout(30.0): + while chunk := await loop.run_in_executor(None, os.read, info_read, 4096): + raw += chunk + with contextlib.suppress(json.JSONDecodeError): + json.loads(raw) + break pid = int(json.loads(raw)["child-pid"]) if raw else 0 if pid: _map_identities(pid) @@ -1212,7 +1221,14 @@ def _session_env(self) -> dict[str, str] | None: return {**os.environ, **self.env} if self.env else None async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> None: - pid = await self.sandbox_pid() + try: + pid = await self.sandbox_pid() + except Exception: + # asyncssh reports a raising process factory to the client as a + # bare "Session request failed" — without this, the reason exists + # nowhere. + LOGGER.exception("session refused: the shared sandbox could not start") + raise # Sessions start from an exact environment, so a terminal's TERM has to # be put there deliberately: without it curses and tput have no # terminal description and fall back or fail outright. From c17a6a730c17113321ca0910d0fcad79f172dac5 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:54:39 -0700 Subject: [PATCH 29/33] refactor(capabilities): delete the SFTP-chroot path emulation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit map_path re-anchored any absolute path outside the capability cwd under it — a client-side emulation of the removed SFTP chroot, kept so legacy prompts addressing workspace files as root-absolute kept landing in the workspace. It made the file helpers disagree with the shell about what a path names: sessions run inside the namespace where the workspace is a real mount, so bash's /etc/hosts was the real file while the edit tool's was /etc/hosts. For a workspace serving a full container filesystem the rewrite corrupted every out-of-workspace path. Paths now reach the session verbatim: relative resolves against the session cwd, absolute means what it says, and the namespace the session runs in is the only path truth. cwd_aliases existed solely to keep the emulation from double-anchoring a symlinked spelling, and goes with it. --- .../openai_compatible/tools/filesystem.py | 11 +-- .../tests/test_provider_native_tools.py | 71 ++++--------------- hud/capabilities/base.py | 14 +--- hud/capabilities/ssh.py | 49 ++----------- hud/environment/tests/test_workspace.py | 44 ++---------- hud/environment/workspace.py | 8 --- 6 files changed, 31 insertions(+), 166 deletions(-) diff --git a/hud/agents/openai_compatible/tools/filesystem.py b/hud/agents/openai_compatible/tools/filesystem.py index bc1ba6c65..af9305dd3 100644 --- a/hud/agents/openai_compatible/tools/filesystem.py +++ b/hud/agents/openai_compatible/tools/filesystem.py @@ -66,9 +66,6 @@ async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: path = arguments.get("filePath") if not isinstance(path, str) or not path: raise ValueError("filePath is required") - # Map once so the directory predicate and the file read agree on - # the same workspace-anchored path. - path = self.client.map_path(path) offset = _read_offset(arguments.get("offset")) limit = _positive_int(arguments.get("limit"), default=DEFAULT_READ_LIMIT, name="limit") if not (await self.bash(f"test -d {shlex.quote(path)}")).isError: @@ -196,9 +193,6 @@ async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: path = arguments.get("filePath") if not isinstance(path, str) or not path: raise ValueError("filePath is required") - # Map once so existence checks, mkdir, and the write all target - # the same workspace-anchored path. - path = self.client.map_path(path) old = arguments.get("oldString") new = arguments.get("newString") if not isinstance(old, str): @@ -258,7 +252,6 @@ async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: path = arguments.get("filePath") if not isinstance(path, str) or not path: raise ValueError("filePath is required") - path = self.client.map_path(path) content = arguments.get("content") if not isinstance(content, str): raise ValueError("content is required") @@ -294,7 +287,7 @@ async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: pattern = arguments.get("pattern") if not isinstance(pattern, str): raise ValueError("pattern is required") - path = self.client.map_path(str(arguments.get("path") or ".")) + path = str(arguments.get("path") or ".") cmd = f"grep -rn {shlex.quote(pattern)} {shlex.quote(path)}" include = arguments.get("include") if isinstance(include, str) and include: @@ -318,7 +311,7 @@ async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: pattern = arguments.get("pattern") if not isinstance(pattern, str): raise ValueError("pattern is required") - path = self.client.map_path(str(arguments.get("path") or ".")) + path = str(arguments.get("path") or ".") return await self.bash(f"find {shlex.quote(path)} -name {shlex.quote(pattern)}") diff --git a/hud/agents/tests/test_provider_native_tools.py b/hud/agents/tests/test_provider_native_tools.py index 014990f79..fae87d404 100644 --- a/hud/agents/tests/test_provider_native_tools.py +++ b/hud/agents/tests/test_provider_native_tools.py @@ -319,54 +319,27 @@ async def test_openai_compatible_write_stores_file_via_ssh_exec() -> None: assert ssh.files["/REPORT.md"] == b"done" -async def test_absolute_paths_anchor_to_the_capability_cwd() -> None: - """The old SFTP chroot resolved ``/REPORT.md`` against the workspace root; - exec-channel file helpers must keep that contract via the capability cwd.""" - ssh = _FakeSSH(cwd="/workspace", files={"/workspace/f.txt": b"inside"}) +async def test_paths_reach_the_session_verbatim() -> None: + """A path means what it says in the session's own namespace: file helpers + and shell commands must never disagree about what a path names, so + nothing is anchored or rewritten on the way through.""" + ssh = _FakeSSH(cwd="/app", files={"/app/f.txt": b"inside"}) tool = WriteTool(spec=WriteTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - await tool.execute({"filePath": "/REPORT.md", "content": "done"}) + await tool.execute({"filePath": "/tmp/probe.txt", "content": "done"}) - assert ssh.files["/workspace/REPORT.md"] == b"done" - # Paths already inside the workspace are untouched. - assert await cast("SSHClient", ssh).read_text("/workspace/f.txt") == "inside" - - -def test_map_path_clamps_traversal_like_a_chroot() -> None: - ssh = cast("SSHClient", _FakeSSH(cwd="/workspace")) - assert ssh.map_path("/workspace/../etc/passwd") == "/workspace/etc/passwd" - assert ssh.map_path("/../etc/passwd") == "/workspace/etc/passwd" - assert ssh.map_path("../../etc/passwd") == "/workspace/etc/passwd" - assert ssh.map_path("a/../b.txt") == "/workspace/b.txt" - assert ssh.map_path("/") == "/workspace" - assert ssh.map_path(".") == "/workspace" - - -def test_map_path_handles_windows_native_paths() -> None: - """The workspace publishes cwd via as_posix(); callers pass native - backslash paths, and NTFS compares case-insensitively.""" - cap = Capability( - name="shell", - protocol="ssh/2", - url="ssh://localhost:22", - params={"shell": "powershell", "cwd": "C:/work"}, - ) - ssh = SSHClient(cap, cast("Any", None)) - assert ssh.map_path("C:\\work\\file.txt") == "C:/work/file.txt" - assert ssh.map_path("C:\\Work\\sub\\f.txt") == "C:/work/sub/f.txt" - assert ssh.map_path("D:\\other\\f.txt") == "C:/work/other/f.txt" - assert ssh.map_path("\\temp\\f.txt") == "C:/work/temp/f.txt" - assert ssh.map_path("sub\\f.txt") == "C:/work/sub/f.txt" - assert ssh.map_path("C:\\work\\..\\secrets.txt") == "C:/work/secrets.txt" + assert ssh.files["/tmp/probe.txt"] == b"done" + assert "/app/tmp/probe.txt" not in ssh.files + assert await cast("SSHClient", ssh).read_text("/app/f.txt") == "inside" async def test_read_maps_the_directory_predicate_and_listing_together() -> None: - """`test -d`, listing, and reads must agree on the anchored path, or - absolute workspace dirs are misclassified as files.""" + """`test -d`, listing, and reads must agree on the same path, or + workspace dirs are misclassified as files.""" ssh = _FakeSSH(cwd="/workspace", files={"/workspace/pkg/mod.py": b"x = 1\n"}) tool = ReadTool(spec=ReadTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - result = await tool.execute({"filePath": "/pkg"}) + result = await tool.execute({"filePath": "/workspace/pkg"}) text = result_text(result) assert "directory" in text @@ -538,26 +511,6 @@ async def test_gemini_edit_creates_file_when_old_string_empty() -> None: assert ssh.files["/n.txt"] == b"fresh" -def test_map_path_leaves_a_symlinked_spelling_of_the_workspace_alone() -> None: - """A workspace made at /tmp/w is served as /private/tmp/w on macOS; re-anchoring - the caller's spelling instead of stripping it nests the path under itself.""" - cap = Capability( - name="shell", - protocol="ssh/2", - url="ssh://localhost:22", - params={"cwd": "/private/tmp/w", "cwd_aliases": ["/tmp/w"]}, - ) - ssh = SSHClient(cap, cast("Any", None)) - - assert ssh.map_path("/tmp/w/calc.py") == "/private/tmp/w/calc.py" - assert ssh.map_path("/private/tmp/w/calc.py") == "/private/tmp/w/calc.py" - assert ssh.map_path("/tmp/w") == "/private/tmp/w" - # Workspace-relative addressing still anchors, and an unrelated absolute - # path is still clamped into the workspace like a chroot. - assert ssh.map_path("/REPORT.md") == "/private/tmp/w/REPORT.md" - assert ssh.map_path("/tmp/elsewhere/f.txt") == "/private/tmp/w/tmp/elsewhere/f.txt" - - async def test_reading_a_missing_file_is_a_tool_error_not_a_raised_traceback() -> None: """Reading before creating is the first thing an editor tool does; that failure must come back as a tool result carrying the shell's message.""" diff --git a/hud/capabilities/base.py b/hud/capabilities/base.py index d8d9548b8..bad6565e5 100644 --- a/hud/capabilities/base.py +++ b/hud/capabilities/base.py @@ -7,12 +7,9 @@ import sys from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar, Self +from typing import Any, ClassVar, Self from urllib.parse import urlsplit -if TYPE_CHECKING: - from collections.abc import Sequence - #: Matches the scheme prefix of a URL (RFC 3986). SCHEME_RE: re.Pattern[str] = re.compile(r"^([a-zA-Z][a-zA-Z0-9+\-.]*):") @@ -82,7 +79,6 @@ def ssh( client_key_path: str | os.PathLike[str] | None = None, shell: str | None = None, cwd: str | None = None, - cwd_aliases: Sequence[str] | None = None, ) -> Capability: """``ssh/2`` — SSH daemon with publickey auth. @@ -93,10 +89,8 @@ def ssh( type (``bash``, ``powershell``, ``cmd``). Defaults to auto-detect from ``sys.platform`` at construction time. Agents read this to format commands correctly. ``cwd`` is the absolute path sessions - start in (the served workspace); clients anchor file paths to it. - ``cwd_aliases`` are other names the same directory answers to (paths - that reach it through symlinks), which clients treat as already - anchored rather than as workspace-relative addresses. + start in. Paths are the session namespace's own — clients pass them + verbatim, and nothing is anchored or rewritten. """ normalized = normalize_url(url, default_scheme="ssh", default_port=22) if shell is None: @@ -108,8 +102,6 @@ def ssh( params["client_key_path"] = os.fspath(client_key_path) if cwd is not None: params["cwd"] = cwd - if cwd_aliases: - params["cwd_aliases"] = list(cwd_aliases) return cls(name=name, protocol="ssh/2", url=normalized, params=params) @classmethod diff --git a/hud/capabilities/ssh.py b/hud/capabilities/ssh.py index a07fbed54..314218595 100644 --- a/hud/capabilities/ssh.py +++ b/hud/capabilities/ssh.py @@ -3,7 +3,6 @@ from __future__ import annotations import base64 -import posixpath import shlex from typing import Any, ClassVar, Self from urllib.parse import urlsplit @@ -14,7 +13,13 @@ class SSHClient(CapabilityClient): - """Thin asyncssh wrapper. Exposes the raw connection via ``conn``.""" + """Thin asyncssh wrapper. Exposes the raw connection via ``conn``. + + File helpers pass paths to the session verbatim: relative paths resolve + against the session cwd, absolute paths mean what they say. The namespace + the session runs in is the only path truth — file helpers and shell + commands must never disagree about what a path names. + """ protocol: ClassVar[str] = "ssh/2" @@ -48,46 +53,8 @@ def conn(self) -> asyncssh.SSHClientConnection: """Raw asyncssh connection for commands and port forwarding.""" return self._conn - def map_path(self, path: str) -> str: - """Anchor a path to the session cwd (the served workspace). - - The old SFTP subsystem was chrooted, so harness tools address files as - ``/REPORT.md`` meaning workspace-relative. The exec channel sees the - real filesystem; replicate the chroot: strip the cwd prefix if present, - normalize the remainder against ``/`` (clamping ``..`` at the root, - exactly as a chroot does), and re-anchor under the cwd. Idempotent. - - ``cwd_aliases`` are the same directory reached through symlinks, and are - stripped like the cwd: re-anchoring one would turn an already correct - absolute path into a nested one that does not exist. - """ - cwd = str(self.capability.params.get("cwd", "")).rstrip("/") - if not cwd: - return path - aliases = [ - str(alias).rstrip("/") for alias in self.capability.params.get("cwd_aliases") or [] - ] - if self._is_windows: - # The workspace publishes cwd via as_posix() (e.g. "C:/work") but - # callers pass native paths ("C:\work\file.txt"); NTFS paths are - # case-insensitive. - path = path.replace("\\", "/") - if path.lower() == cwd.lower() or path.lower().startswith(cwd.lower() + "/"): - path = path[len(cwd) :] - elif len(path) >= 2 and path[1] == ":" and path[0].isalpha(): - # Drive-absolute outside the workspace: anchor like the chroot. - path = path[2:] - else: - for prefix in (cwd, *aliases): - if prefix and (path == prefix or path.startswith(prefix + "/")): - path = path[len(prefix) :] - break - normalized = posixpath.normpath("/" + path.lstrip("/")) - return cwd if normalized == "/" else cwd + normalized - async def read_text(self, path: str) -> str: """Read a UTF-8 text file through the exec channel.""" - path = self.map_path(path) if self._is_windows: quoted = _powershell_quote(path) script = f"[Convert]::ToBase64String([IO.File]::ReadAllBytes({quoted}))" @@ -100,7 +67,6 @@ async def read_text(self, path: str) -> str: async def write_text(self, path: str, content: str) -> None: """Write UTF-8 text through the exec channel without command interpolation.""" - path = self.map_path(path) if self._is_windows: quoted = _powershell_quote(path) truncate = f"[IO.File]::WriteAllBytes({quoted},[byte[]]@())" @@ -120,7 +86,6 @@ async def write_text(self, path: str, content: str) -> None: async def listdir(self, path: str) -> list[str]: """List direct children through the exec channel.""" - path = self.map_path(path) if self._is_windows: script = f"Get-ChildItem -Force -Name -LiteralPath {_powershell_quote(path)}" result = await self._conn.run(_powershell(script), check=True) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index e92be76af..f2f51dc80 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -83,11 +83,13 @@ async def test_file_operations_use_the_exec_channel(tmp_path: Path) -> None: await client.write_text("hello world.txt", "héllo\n") assert await client.read_text("hello world.txt") == "héllo\n" assert await client.listdir(".") == ["hello world.txt"] - # Absolute paths anchor to the workspace, like the old SFTP chroot. - await client.write_text("/REPORT.md", "done") - assert (tmp_path / "root" / "REPORT.md").read_text() == "done" - assert await client.read_text("/REPORT.md") == "done" - assert "REPORT.md" in await client.listdir("/") + # Absolute paths mean what they say in the session's namespace — + # never re-anchored under the workspace. + outside = tmp_path / "outside.txt" + await client.write_text(str(outside), "done") + assert outside.read_text() == "done" + assert await client.read_text(str(outside)) == "done" + assert not (tmp_path / "root" / str(outside).lstrip("/")).exists() finally: await ws.stop() @@ -743,38 +745,6 @@ def test_required_isolation_refuses_when_unavailable(monkeypatch, tmp_path) -> N ws.Workspace(tmp_path, require_isolation=True) -@pytest.mark.asyncio -async def test_a_symlinked_root_publishes_both_spellings(tmp_path: Path) -> None: - """A workspace addressed through a symlink (macOS /tmp -> /private/tmp) serves the - real path, so it must publish the caller's spelling too or clients re-anchor it.""" - real = tmp_path / "real" - real.mkdir() - link = tmp_path / "link" - link.symlink_to(real, target_is_directory=True) - - ws = Workspace(link) - await ws.start() - try: - cap = ws.capability() - assert cap.params["cwd"] == real.as_posix() - assert cap.params["cwd_aliases"] == [link.as_posix()] - - client = SSHClient(cap, cast("Any", None)) - assert client.map_path(f"{link}/calc.py") == f"{real}/calc.py" - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_a_plain_root_publishes_no_alias(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - await ws.start() - try: - assert "cwd_aliases" not in ws.capability().params - finally: - await ws.stop() - - @pytest.mark.asyncio async def test_identity_map_reads_a_chunked_info_document() -> None: """bwrap's info JSON arrives in as many chunks as the pipe delivers; a diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 830c73ab8..976f306cb 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -394,13 +394,6 @@ def __init__( # Only override the default; respect an explicit guest_path. if self._bwrap is None and guest_path == "/workspace": self._guest_path = self.root.as_posix() - # The caller's spelling of the same directory when it differs from the real - # path (macOS resolves /tmp to /private/tmp) and sessions run under the real - # one. A client that knows only the real path treats the other spelling as a - # workspace-relative address and re-anchors it somewhere that does not exist. - given = Path(root).absolute().as_posix() - real = self.root.as_posix() - self._cwd_aliases = [given] if given != real and self._guest_path == real else [] # ssh config self._ssh_host = host self._ssh_port = port @@ -684,7 +677,6 @@ def capability(self, name: str = "shell") -> Capability: client_key=key_path.read_text() if key_path else None, client_key_path=key_path, cwd=self._guest_path, - cwd_aliases=self._cwd_aliases or None, ) @property From c3b20a8bb203ed871b3fa9cce7824705e871b5a7 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:21:30 -0700 Subject: [PATCH 30/33] fix(eval): exclude errored runs from the job mean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Infrastructure failure is never a score: a run whose trace or grade ended in error carries no verdict, and averaging it in as a zero silently deflates the job — a sweep with failing provisioning reads as a worse model. Errored runs are excluded from Job.reward and surfaced on Job.errors instead. --- hud/eval/job.py | 21 ++++++++++++++++++--- hud/eval/tests/test_job.py | 26 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/hud/eval/job.py b/hud/eval/job.py index 0161461c7..56a079512 100644 --- a/hud/eval/job.py +++ b/hud/eval/job.py @@ -56,10 +56,25 @@ async def start(cls, name: str, *, group: int = 1, taskset_id: str | None = None @property def reward(self) -> float: - """Mean reward across runs (0.0 for an empty job).""" - if not self.runs: + """Mean reward across graded runs (0.0 when none were graded). + + An errored run carries no verdict — infrastructure failure is never + a score — so it is excluded from the mean rather than averaged in as + a zero that silently deflates the job. :attr:`errors` holds them. + """ + graded = [run.reward for run in self.runs if not (run.trace.is_error or run.grade.is_error)] + if not graded: return 0.0 - return sum(run.reward for run in self.runs) / len(self.runs) + return sum(graded) / len(graded) + + @property + def errors(self) -> list[Run]: + """Runs that ended in error: ungraded, and excluded from :attr:`reward`. + + Either error signal counts — a trace that ended in error (launch or + mid-run failure) or a grade the env itself marked as an error. + """ + return [run for run in self.runs if run.trace.is_error or run.grade.is_error] @property def results(self) -> dict[str, list[Run]]: diff --git a/hud/eval/tests/test_job.py b/hud/eval/tests/test_job.py index d1b5731ba..f3f67afbd 100644 --- a/hud/eval/tests/test_job.py +++ b/hud/eval/tests/test_job.py @@ -86,3 +86,29 @@ async def test_trace_exit_omits_metadata_when_extra_empty(recorder: _Recorder) - assert len(recorder.calls) == 1 _, body = recorder.calls[0] assert "metadata" not in body + + +def test_errored_runs_do_not_deflate_the_job_reward() -> None: + """Infrastructure failure is never a score: a run that errored (a launch + failure, or a hosted trace that ended in error) carries no verdict and + must not drag the job mean down as a silent zero.""" + from hud.eval.job import Job + from hud.eval.run import Grade + + graded = _run_with("t1", extra={}) + graded.grade = Grade(reward=1.0) + failed = Run.failed("provisioning never finished") + + job = Job(id="j1", name="test", runs=[graded, failed]) + + assert job.reward == 1.0 + assert job.errors == [failed] + + +def test_job_with_only_errors_reports_zero_reward() -> None: + from hud.eval.job import Job + + job = Job(id="j2", name="test", runs=[Run.failed("boom")]) + + assert job.reward == 0.0 + assert job.errors and job.errors[0].trace.is_error From a85e364db7a65dc6bc71d720b54310c83e4d457f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:51:48 -0700 Subject: [PATCH 31/33] fix(environment): wait on setsid when it must fork Spawned without a sandbox, setsid is already a process-group leader and forks; without --wait the parent exits immediately, the server takes that for session end, and the channel closes before the payload speaks. Inside the sandbox setsid execs in place and --wait is inert. --- hud/environment/workspace.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 976f306cb..8f490abb2 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -268,12 +268,14 @@ def _ctty_argv() -> list[str]: tty file descriptors alone are not a terminal session: ``/dev/tty`` cannot be opened, and job control has no foreground process group to signal. Run inside the sandbox, ``setsid`` is not a process-group leader there and so - execs in place rather than forking, which keeps the pid and exit status - the caller is waiting on. Where the binary is absent (macOS ships none) - the session still gets a working tty, only without a ctty. + execs in place. Spawned directly (no sandbox), it *is* already a leader + and must fork — ``--wait`` keeps the parent alive relaying the payload's + exit status, or the session would end the instant the fork returns. + Where the binary is absent (macOS ships none) the session still gets a + working tty, only without a ctty. """ setsid = shutil.which("setsid") - return [setsid, "-c"] if setsid else [] + return [setsid, "--wait", "-c"] if setsid else [] async def _pty_streams(master_fd: int) -> tuple[Any, asyncio.StreamReader]: From de06e6660ab912325cdce0947de0b01b80322fc7 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:57:57 -0700 Subject: [PATCH 32/33] fix(environment): give an isolated command the image's environment Both branches of Workspace.run claim to run a command against the same workspace, but the isolated one built its environment from nothing: a no-network Harbor verifier ran test.sh under env -i with no PATH, losing the interpreters and tools the image installed, while the same verifier with network inherited the whole container environment. A task's declared isolation must not change what its verifier can find. --- hud/environment/tests/test_workspace.py | 36 +++++++++++++++++++++++++ hud/environment/workspace.py | 7 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index f2f51dc80..57ed84954 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -460,6 +460,30 @@ async def test_run_can_use_a_fresh_no_network_sandbox( assert len(kwargs["pass_fds"]) == 2 +@pytest.mark.asyncio +async def test_an_isolated_command_keeps_the_image_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A command must not lose the image's PATH — the interpreters and tools + the task installed — merely because it asked for an isolated sandbox. + Both branches of run() give it the serving environment, less HUD's own.""" + monkeypatch.setenv("PATH", "/task/bin:/usr/bin") + monkeypatch.setenv("HUD_API_KEY", "sk-secret") + ws = Workspace(tmp_path / "root") + monkeypatch.setattr(ws, "_bwrap", "/usr/bin/bwrap") + monkeypatch.setattr(workspace_mod, "install_identity_map", AsyncMock(return_value=9)) + complete = AsyncMock(return_value=ProcessResult(0, b"", b"")) + spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) + monkeypatch.setattr(workspace_mod, "create_process_group_exec", spawn) + + await ws.run(["test.sh"], isolated=True, identity=None) + + assert spawn.await_args is not None + argv, _ = spawn.await_args + assert "PATH=/task/bin:/usr/bin" in argv + assert not any(arg.startswith("HUD_API_KEY=") for arg in argv) + + def test_a_peer_answers_at_the_address_the_task_expects() -> None: """A task that names a service says where it expects to find it. Placed anywhere else, the task's own client configuration points at nothing.""" @@ -773,3 +797,15 @@ def write_in_chunks() -> None: for fd in (info_read, block_read, block_write): with contextlib.suppress(OSError): os.close(fd) + + +def test_the_proxy_refuses_to_forward_a_folded_header() -> None: + """An upstream header is remote text, and a folded value keeps its CRLF + through http.client. Re-emitting it verbatim would let an allowed host + write headers of its own into the response the workspace reads.""" + from hud.environment.egress import _sanitized + + assert _sanitized("text/plain") == "text/plain" + assert _sanitized("a\r\n b") is None + assert _sanitized("a\nX-Injected: 1") is None + assert _sanitized(None) is None diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 8f490abb2..f6daad7d3 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -768,7 +768,12 @@ async def run( *self.bwrap_argv( [*drop, *command], env=process_env, - inherit_host_env=False, + # Same environment the joined branch gives a command (the + # serving process's, less HUD's own): a command must not + # run without the image's PATH — losing the interpreters + # and tools the task installed — merely because it asked + # for an isolated sandbox. + inherit_host_env=True, inherit_workspace_env=inherit_workspace_env, info_fd=info_write, userns_block_fd=block_read, From 2d6344c735f7e12ea3c73882bb369979b145e84b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:02:12 -0700 Subject: [PATCH 33/33] fix(environment): relay only headers the HTTP grammar admits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An upstream response is remote text and http.client preserves a folded value's CRLF, so a header relayed verbatim can carry headers of its own into the response the workspace reads. Names and values are matched against RFC 9110's grammar — a positive definition of what a header may contain, rather than a guess at which characters an attacker would use — and a field outside it fails the whole response with 502 rather than being quietly repaired: a proxy that rewrites responses is worse to debug than one that says it could not relay this one. Fields are validated before any are written, since half a status line has already committed the connection by the time a later header fails. --- hud/environment/egress.py | 68 +++++++++++++++++++++---- hud/environment/tests/test_workspace.py | 25 +++++---- 2 files changed, 73 insertions(+), 20 deletions(-) diff --git a/hud/environment/egress.py b/hud/environment/egress.py index 20c2494e5..887bac1b6 100644 --- a/hud/environment/egress.py +++ b/hud/environment/egress.py @@ -28,6 +28,7 @@ import json import logging import os +import re import select import shutil import socket @@ -65,6 +66,35 @@ } ) + +#: RFC 9110 grammar for what a header may contain. A name is a `token`; a +#: value is visible ASCII, obs-text, and blanks — no control characters, so +#: nothing in a relayed header can end it and begin another. Matched +#: positively: a proxy relays what the grammar admits, rather than guessing +#: which characters an attacker would have used. +_FIELD_NAME = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") +_FIELD_VALUE = re.compile(r"[\t\x20-\x7e\x80-\xff]*") + + +class _Unrelayable(ValueError): + """An upstream response that cannot be relayed as the upstream sent it.""" + + +def _field(name: str, value: str) -> tuple[str, str]: + """*name* and *value* as a header, or raise. + + An upstream response is remote text, and ``http.client`` preserves a + folded value's CRLF, so a header relayed verbatim can carry headers of + its own into the response the workspace reads. Refusing beats repairing: + a value that has to be altered to be safe is no longer the value the + upstream sent, and a proxy that quietly rewrites responses is worse to + debug than one that says it could not relay this one. + """ + if not _FIELD_NAME.fullmatch(name) or not _FIELD_VALUE.fullmatch(value): + raise _Unrelayable(f"header {name[:32]!r} is not relayable") + return name, value + + #: The proxy port offered on the workspace's loopback. 3128 is unremarkable — #: an egress proxy is ordinary infrastructure, unlike a control channel. BRIDGE_PORT = 3128 @@ -234,14 +264,17 @@ class _Proxy(BaseHTTPRequestHandler): def log_message(self, *_: object) -> None: """The workspace's traffic is not the substrate's log.""" - def _deny(self) -> None: - # Loud and diagnosable from inside the workspace: a host held back by - # policy should not look like a network that is merely broken. - self.send_response(403) - self.send_header("X-Proxy-Error", "blocked-by-allowlist") + def _fail(self, status: int, reason: str) -> None: + # Loud and diagnosable from inside the workspace: a request the proxy + # would not carry should not look like a network that is merely broken. + self.send_response(status) + self.send_header("X-Proxy-Error", reason) self.send_header("Content-Length", "0") self.end_headers() + def _deny(self) -> None: + self._fail(403, "blocked-by-allowlist") + def do_CONNECT(self) -> None: host, _, port = self.path.rpartition(":") if not permitted(host, self.allowed): @@ -274,19 +307,32 @@ def _forward(self) -> None: try: connection.request(self.command, path, body=body, headers=headers) response = connection.getresponse() - self.send_response(response.status, response.reason) + # Validate every field before writing any of them: a response is + # relayed whole or not at all, and half a status line has already + # committed this connection by the time a later header fails. + relayed = [ + _field(key, value) + for key, value in response.getheaders() + if key.lower() not in _HOP_BY_HOP and key.lower() != "content-length" + ] length = response.getheader("Content-Length") - for key, value in response.getheaders(): - if key.lower() not in _HOP_BY_HOP and key.lower() != "content-length": - self.send_header(key, value) - if length is not None: - self.send_header("Content-Length", length) + framed = length is not None and length.strip().isdigit() + _field("Reason", response.reason or "") + self.send_response(response.status, response.reason) + for key, value in relayed: + self.send_header(key, value) + if framed: + assert length is not None + self.send_header("Content-Length", length.strip()) else: # Nothing upstream framed the body, so the close delimits it. self.send_header("Connection", "close") self.close_connection = True self.end_headers() shutil.copyfileobj(response, self.wfile) + except _Unrelayable as error: + LOGGER.warning("refusing to relay %s: %s", parts.hostname, error) + self._fail(502, "unrelayable-upstream-header") except (OSError, http.client.HTTPException): self.close_connection = True finally: diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 57ed84954..b98a97d2b 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -22,6 +22,7 @@ from hud.capabilities import SSHClient from hud.environment import workspace as workspace_mod +from hud.environment.egress import _field, _Unrelayable from hud.environment.workspace import Mount, Workspace from hud.utils.process import ProcessResult @@ -799,13 +800,19 @@ def write_in_chunks() -> None: os.close(fd) -def test_the_proxy_refuses_to_forward_a_folded_header() -> None: +def test_the_proxy_refuses_to_relay_a_header_it_cannot_represent() -> None: """An upstream header is remote text, and a folded value keeps its CRLF - through http.client. Re-emitting it verbatim would let an allowed host - write headers of its own into the response the workspace reads.""" - from hud.environment.egress import _sanitized - - assert _sanitized("text/plain") == "text/plain" - assert _sanitized("a\r\n b") is None - assert _sanitized("a\nX-Injected: 1") is None - assert _sanitized(None) is None + through http.client. Relayed verbatim it would carry headers of its own + into the response the workspace reads, so a field outside the grammar + fails the whole response rather than being quietly repaired.""" + assert _field("Content-Type", "text/plain") == ("Content-Type", "text/plain") + assert _field("X-Meta", "") == ("X-Meta", "") + for name, value in ( + ("X-Evil", "a\r\n b"), + ("X-Evil", "a\nX-Injected: 1"), + ("X-Evil", "a\x00b"), + ("Bad Name", "fine"), + ("X-Evil\r\nX-Injected", "fine"), + ): + with pytest.raises(_Unrelayable): + _field(name, value)