From 316e9be3939bf37c92998d091afff68ea63e2da2 Mon Sep 17 00:00:00 2001 From: agentforce314 <273884145+agentforce314@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:27:38 -0700 Subject: [PATCH 1/5] Revert "Revert "Merge pull request #897 from agentforce314/feat/nano-bash-no-timeout"" This reverts commit b21a6baa732c4a0b735c72724243a9abcd56cbf5. --- eval/harbor/run_tb21_nano_max.sh | 1 + src/nano/prompt.py | 3 ++ src/nano/tool_docs.py | 7 +++- src/tool_system/tools/bash/bash_tool.py | 24 +++++++++-- tests/nano/test_nano_bash.py | 56 +++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/nano/test_nano_bash.py diff --git a/eval/harbor/run_tb21_nano_max.sh b/eval/harbor/run_tb21_nano_max.sh index 41ca96cc..91475904 100755 --- a/eval/harbor/run_tb21_nano_max.sh +++ b/eval/harbor/run_tb21_nano_max.sh @@ -83,6 +83,7 @@ PYTHONPATH="$ROOT/eval/harbor" harbor run \ --ak "source=$WHEEL" \ --ak nano=1 \ --ak effort=max \ + --ak "max_turns=${NANO_MAX_TURNS:-600}" \ ${AK_EXTRA[@]+"${AK_EXTRA[@]}"} echo diff --git a/src/nano/prompt.py b/src/nano/prompt.py index 0576ee41..eefeddc6 100644 --- a/src/nano/prompt.py +++ b/src/nano/prompt.py @@ -76,6 +76,9 @@ "files merely by opening them", "- Get a minimal working version of the requested deliverable in " "place early, then iterate to improve it", + "- Run long builds, training runs, or downloads as ONE blocking bash " + "command (tee output to a log) — never launch with nohup and poll in " + "a loop", "- Before finishing, re-read the task and verify each explicit " "requirement against what you actually produced; plausible output " "is not verified output", diff --git a/src/nano/tool_docs.py b/src/nano/tool_docs.py index 05c36d38..2043b6b1 100644 --- a/src/nano/tool_docs.py +++ b/src/nano/tool_docs.py @@ -22,8 +22,11 @@ ), "Bash": ( "Executes a bash command and returns stdout+stderr. The working " - "directory persists between commands; shell state does not. Long " - "output is truncated. Quote paths containing spaces." + "directory persists between commands; shell state does not. No " + "default timeout: a long build or training run can block in a " + "single call (tee output to a log file) — pass timeout (ms) only " + "when you want one. Long output is truncated. Quote paths " + "containing spaces." ), # Edit is intentionally absent: the nano registry swaps in # NanoEditTool (src/nano/edit_tool.py), which carries its own doc for diff --git a/src/tool_system/tools/bash/bash_tool.py b/src/tool_system/tools/bash/bash_tool.py index f6aecb44..84b6f6db 100644 --- a/src/tool_system/tools/bash/bash_tool.py +++ b/src/tool_system/tools/bash/bash_tool.py @@ -659,19 +659,37 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: ) # Resolve timeout: prefer explicit timeout (ms), fall back to timeout_s (legacy), then default + # + # Nano mode removes the ceiling entirely (pi's bash has no default + # timeout). Measured on TB 2.1 (tb21-nano-flash-max-2): the 2-minute + # default / 10-minute cap forced nohup-and-poll loops on every long + # build or training run — 225 poll-pattern commands across the 25 + # failed tasks, with compile-compcert alone burning 111 polls and + # dying at the 300-turn ceiling mid-build. One blocking call costs + # zero turns; the benchmark/task wall-clock and the abort signal + # (checked continuously by _run_supervised) remain the backstops. + from src.nano.state import is_nano_mode as _bash_is_nano + + _nano = _bash_is_nano() + _NANO_NO_TIMEOUT_S = 86_400 # 24h — effectively "until abort/wall-clock" timeout_ms = tool_input.get("timeout") if timeout_ms is not None: max_ms = get_max_timeout_ms() if not isinstance(timeout_ms, (int, float)) or timeout_ms < 1000: raise ToolInputError("timeout must be at least 1000 ms") - if timeout_ms > max_ms: + if timeout_ms > max_ms and not _nano: raise ToolInputError(f"timeout must not exceed {max_ms} ms") timeout_s = int(timeout_ms / 1000) else: timeout_s = tool_input.get("timeout_s") if timeout_s is None: - timeout_s = int(get_default_timeout_ms() / 1000) - if not isinstance(timeout_s, int) or timeout_s < 1 or timeout_s > 600: + timeout_s = ( + _NANO_NO_TIMEOUT_S if _nano + else int(get_default_timeout_ms() / 1000) + ) + if not isinstance(timeout_s, int) or timeout_s < 1 or ( + timeout_s > 600 and not _nano + ): raise ToolInputError("timeout_s must be an integer between 1 and 600") # Persist cwd across invocations (port of ``typescript/src/utils/Shell.ts``, diff --git a/tests/nano/test_nano_bash.py b/tests/nano/test_nano_bash.py new file mode 100644 index 00000000..3ec1ee7b --- /dev/null +++ b/tests/nano/test_nano_bash.py @@ -0,0 +1,56 @@ +"""Nano bash: no timeout ceiling (pi parity for long-running work). + +TB 2.1 evidence (tb21-nano-flash-max-2): the 2-minute default / 10-minute +cap forced nohup-and-poll loops — 225 poll-pattern commands across the 25 +failed tasks; compile-compcert burned 111 polls and hit the 300-turn +ceiling mid-build. Under nano a long build blocks in ONE call; the abort +signal and the task wall-clock remain the backstops. Stock behavior is +byte-identical. +""" + +from __future__ import annotations + +import pytest + +from src.nano.state import set_nano_mode +from src.tool_system.errors import ToolInputError +from src.tool_system.tools import BashTool + + +@pytest.fixture +def ctx(tmp_path): + from src.tool_system.context import ToolContext + + return ToolContext(cwd=tmp_path, workspace_root=tmp_path) + + +def _run(ctx, **extra): + return BashTool.call({"command": "echo long-ok", **extra}, ctx) + + +def test_nano_accepts_timeouts_beyond_the_stock_cap(ctx): + set_nano_mode(True) + result = _run(ctx, timeout=1_200_000) # 20 minutes + assert "long-ok" in str(result.output) + + +def test_stock_still_rejects_beyond_cap(ctx): + with pytest.raises(ToolInputError, match="must not exceed"): + _run(ctx, timeout=1_200_000) + + +def test_nano_default_runs_without_explicit_timeout(ctx): + set_nano_mode(True) + result = _run(ctx) + assert "long-ok" in str(result.output) + + +def test_stock_default_unchanged(ctx): + result = _run(ctx) + assert "long-ok" in str(result.output) + + +def test_minimum_floor_still_enforced_in_nano(ctx): + set_nano_mode(True) + with pytest.raises(ToolInputError, match="at least 1000 ms"): + _run(ctx, timeout=10) From d3c3658ca3aa517ff90665f38ba9faca2ae51cf8 Mon Sep 17 00:00:00 2001 From: agentforce314 <273884145+agentforce314@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:27:38 -0700 Subject: [PATCH 2/5] Revert "Revert "Merge pull request #898 from agentforce314/feat/nano-constraint-checklist"" This reverts commit 3e705e4af081a22594b7346c52c55bbc2c8942c8. --- src/nano/prompt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nano/prompt.py b/src/nano/prompt.py index eefeddc6..b3dbb961 100644 --- a/src/nano/prompt.py +++ b/src/nano/prompt.py @@ -79,9 +79,10 @@ "- Run long builds, training runs, or downloads as ONE blocking bash " "command (tee output to a log) — never launch with nohup and poll in " "a loop", - "- Before finishing, re-read the task and verify each explicit " - "requirement against what you actually produced; plausible output " - "is not verified output", + "- Before finishing, re-read the task and list every explicit " + "requirement and constraint (exact formats, tolerances, numeric " + "bounds, required sources or methods), then test your artifact " + "against each one; plausible output is not verified output", "- Be concise in your responses", "- Show file paths clearly when working with files", ) From cba7546039f459f4aab40bd431990b3759449d2d Mon Sep 17 00:00:00 2001 From: agentforce314 <273884145+agentforce314@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:38:29 -0700 Subject: [PATCH 3/5] feat(nano): drop the run_in_background trap from the Bash schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tb21-nano-flash-max-2 shows 29 run_in_background launches across 9 trials — all long-running-work tasks (compile-compcert, caffe-cifar-10, train-fasttext, ...). In nano the param is a trap: retrieving a backgrounded command's output needs the TaskOutput tool, which the nano surface deliberately lacks, so the model strands the output and falls into exactly the log-poll loops the no-timeout contract eliminates. With no default timeout, one blocking call is strictly better. The nano Bash schema no longer advertises the param (additionalProperties stays false, so passing it anyway yields an actionable validation error naming it), and the legacy timeout_s description loses its stale '(1-600)' cap text. Stock schema untouched. Co-Authored-By: Claude Fable 5 --- src/nano/registry.py | 26 ++++++++++++++++++++++++++ tests/nano/test_nano_bash.py | 12 ++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/nano/registry.py b/src/nano/registry.py index 99f4f49a..a5d070cb 100644 --- a/src/nano/registry.py +++ b/src/nano/registry.py @@ -65,6 +65,8 @@ def build_nano_registry() -> ToolRegistry: doc = NANO_TOOL_DOCS.get(tool.name) if doc is not None: tool = replace(tool, prompt=lambda _doc=doc: _doc) + if tool.name == "Bash": + tool = replace(tool, input_schema=_nano_bash_schema(tool.input_schema)) registry.register(tool) try: from src.tool_system.tools import VisionAnalyzeTool @@ -87,6 +89,30 @@ def build_nano_registry() -> ToolRegistry: return registry +def _nano_bash_schema(schema) -> dict: + """Bash schema minus the background-execution trap. + + ``run_in_background`` needs the TaskOutput tool to retrieve results — + absent from the nano surface, so a backgrounded command strands its + output and forces the poll loops the no-timeout contract exists to + eliminate (tb21-nano-flash-max-2: 29 background launches across 9 + trials, all long-running-work tasks). With no default timeout, one + blocking call is strictly better. The stale "(1-600)" cap text on the + legacy ``timeout_s`` goes too. additionalProperties stays false, so a + model that still passes run_in_background gets an actionable + validation error naming the property. + """ + out = dict(schema) + props = dict(out.get("properties") or {}) + props.pop("run_in_background", None) + if "timeout_s" in props: + ts = dict(props["timeout_s"]) + ts["description"] = "Timeout in seconds" + props["timeout_s"] = ts + out["properties"] = props + return out + + def _nano_websearch_configured() -> bool: """Explicit opt-in for WebSearch on the nano surface. diff --git a/tests/nano/test_nano_bash.py b/tests/nano/test_nano_bash.py index 3ec1ee7b..55dd2f99 100644 --- a/tests/nano/test_nano_bash.py +++ b/tests/nano/test_nano_bash.py @@ -54,3 +54,15 @@ def test_minimum_floor_still_enforced_in_nano(ctx): set_nano_mode(True) with pytest.raises(ToolInputError, match="at least 1000 ms"): _run(ctx, timeout=10) + + +def test_nano_bash_schema_drops_background_trap(): + # run_in_background needs TaskOutput (absent in nano) — a backgrounded + # command strands its output; the nano schema must not advertise it. + from src.nano.registry import build_nano_registry + + bash = next(t for t in build_nano_registry().list_tools() if t.name == "Bash") + assert "run_in_background" not in bash.input_schema["properties"] + assert "1-600" not in str(bash.input_schema["properties"].get("timeout_s", {})) + # Stock schema untouched. + assert "run_in_background" in BashTool.input_schema["properties"] From e116fbc86126fb34ce0f2af186a8f1a8db1fee73 Mon Sep 17 00:00:00 2001 From: agentforce314 <273884145+agentforce314@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:02:53 -0700 Subject: [PATCH 4/5] =?UTF-8?q?feat(nano):=20stuck-command=20detection=20?= =?UTF-8?q?=E2=80=94=20concurrent=20pipe=20drain=20+=20output-idle=20watch?= =?UTF-8?q?dog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the question the no-timeout contract raises: how do we detect a bash call stuck forever? Layered: 1. stdin=DEVNULL (already present) — interactive prompts see EOF and fail fast instead of waiting for input forever. 2. NEW concurrent pipe drain: stdout/stderr are drained by reader threads using raw os.read (returns on ANY available bytes; TextIOWrapper.read(n) blocks until n chars and starved the liveness stamp on drip-feed output). This also fixes a latent deadlock in ALL modes: with communicate()-at-exit, any command writing more than the ~64KB pipe buffer blocked on write and could never exit — the stock hard timeout misreported those as 'timed out'; under an unbounded timeout they would hang outright. 3. NEW output-idle watchdog (nano only): a command with NO output for NANO_BASH_IDLE_TIMEOUT_S (default 600s) while still running is presumed stuck and killed with a teaching error — re-run non-interactively / add progress output / pass an explicit timeout. A noisy 40-minute build never trips it. Stock keeps its hard-timeout model unchanged. 4. Explicit timeout param, the abort signal, and the task wall-clock remain the outer layers. Nano Bash doc states the keep-long-jobs-chatty contract. 11 watchdog/ drain tests (silent-stuck killed in seconds with the teaching message; drip-feed chatty command spared; >64KB outputs complete in both modes; stock unaffected); 201 nano+bash tests pass. Co-Authored-By: Claude Fable 5 --- src/nano/tool_docs.py | 7 +- src/tool_system/tools/bash/bash_tool.py | 150 +++++++++++++++++++----- tests/nano/test_nano_bash.py | 56 +++++++++ 3 files changed, 178 insertions(+), 35 deletions(-) diff --git a/src/nano/tool_docs.py b/src/nano/tool_docs.py index 2043b6b1..cf84d0f4 100644 --- a/src/nano/tool_docs.py +++ b/src/nano/tool_docs.py @@ -24,9 +24,10 @@ "Executes a bash command and returns stdout+stderr. The working " "directory persists between commands; shell state does not. No " "default timeout: a long build or training run can block in a " - "single call (tee output to a log file) — pass timeout (ms) only " - "when you want one. Long output is truncated. Quote paths " - "containing spaces." + "single call — pass timeout (ms) only when you want one. A " + "command with NO output for 10 minutes is killed as stuck, so " + "keep long jobs chatty (verbose flags, tee to a log). Long " + "output is truncated. Quote paths containing spaces." ), # Edit is intentionally absent: the nano registry swaps in # NanoEditTool (src/nano/edit_tool.py), which carries its own doc for diff --git a/src/tool_system/tools/bash/bash_tool.py b/src/tool_system/tools/bash/bash_tool.py index 84b6f6db..b867bb15 100644 --- a/src/tool_system/tools/bash/bash_tool.py +++ b/src/tool_system/tools/bash/bash_tool.py @@ -34,6 +34,10 @@ class _BashRunResult: stderr: str interrupted: bool = False timed_out: bool = False + # Killed by the output-idle watchdog (no stdout/stderr for the idle + # window while still running) — the "stuck forever" detector for + # nano's unbounded-timeout contract. + idle_timed_out: bool = False def _get_abort_signal(context: ToolContext) -> Any: @@ -60,8 +64,9 @@ def _run_bash_with_abort( cwd: str, timeout_s: int, abort_signal: Any | None, + idle_timeout_s: int | None = None, ) -> _BashRunResult: - """Run ``argv`` with abort + timeout supervision. + """Run ``argv`` with abort + timeout + output-idle supervision. Replaces ``subprocess.run(..., timeout=...)``: launches the subprocess in its own session/process group, polls for completion @@ -69,6 +74,18 @@ def _run_bash_with_abort( the whole group (SIGTERM → grace → SIGKILL) when either fires. Returning quickly on abort is what makes ESC feel instant — the previous ``subprocess.run`` had to wait the entire timeout. + + Both pipes are drained CONCURRENTLY by reader threads. This is + load-bearing twice over: (1) with ``communicate()``-at-the-end, any + command writing more than the OS pipe buffer (~64KB) blocked on + write and could never exit — under a hard timeout that surfaced as a + bogus "timed out", under nano's unbounded timeout it would hang + outright; (2) the readers stamp ``last_output``, which powers + ``idle_timeout_s`` — the stuck-command detector for the unbounded + contract: a process that stays silent for the whole idle window + while still running is presumed stuck (waiting for input it will + never get, deadlocked, or hung) and is killed with an actionable + error, while a noisy 40-minute build never trips it. """ # ``stdin=DEVNULL`` matches TS ``Shell.ts`` (stdio[0] = 'pipe' with the @@ -76,12 +93,17 @@ def _run_bash_with_abort( # parent's stdin -- when clawcodex runs in a terminal, that's a TTY, and # scaffolders like ``npm create vite`` see ``isatty(0)`` and try to prompt # for confirmation, hanging the command until timeout. + # Byte pipes, decoded at assembly: the drain threads must use raw + # ``os.read``, which returns as soon as ANY bytes are available — + # ``TextIOWrapper.read(n)`` blocks until n chars accumulate, which + # would starve the idle watchdog's liveness stamp on drip-feed output + # (observed: a command echoing every 2s was idle-killed because its + # small writes sat inside the buffered reader). popen_kwargs: dict[str, Any] = { "cwd": cwd, "stdin": subprocess.DEVNULL, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE, - "text": True, **popen_tree_kwargs(), } @@ -95,9 +117,40 @@ def _run_bash_with_abort( proc = subprocess.Popen(argv, **popen_kwargs) + import threading as _threading + + out_chunks: list[bytes] = [] + err_chunks: list[bytes] = [] + last_output = [_time_mod.monotonic()] + + def _drain(stream: Any, chunks: list[bytes]) -> None: + try: + fd = stream.fileno() + while True: + chunk = _os_mod.read(fd, 65536) + if not chunk: + break + chunks.append(chunk) + last_output[0] = _time_mod.monotonic() + except (OSError, ValueError): + pass + finally: + try: + stream.close() + except OSError: + pass + + readers = [ + _threading.Thread(target=_drain, args=(proc.stdout, out_chunks), daemon=True), + _threading.Thread(target=_drain, args=(proc.stderr, err_chunks), daemon=True), + ] + for r in readers: + r.start() + deadline = _time_mod.monotonic() + timeout_s interrupted = False timed_out = False + idle_timed_out = False while True: if proc.poll() is not None: @@ -105,9 +158,13 @@ def _run_bash_with_abort( if abort_signal is not None and getattr(abort_signal, "aborted", False): interrupted = True break - if _time_mod.monotonic() >= deadline: + now = _time_mod.monotonic() + if now >= deadline: timed_out = True break + if idle_timeout_s is not None and now - last_output[0] >= idle_timeout_s: + idle_timed_out = True + break _time_mod.sleep(_ABORT_POLL_INTERVAL_S) # Mirrors TS ``ShellCommand.ts:337-343`` (``#doKill``): both the @@ -120,48 +177,38 @@ def _run_bash_with_abort( # ``interrupted`` / ``timed_out`` are the source-of-truth # discriminator for downstream callers; the exit-code label is # rewritten in ``_bash_call``. - if interrupted or timed_out: + if interrupted or timed_out or idle_timed_out: kill_process_tree(proc.pid, force=True) try: proc.wait(timeout=_KILL_REAP_TIMEOUT_S) except subprocess.TimeoutExpired: # SIGKILL is uncatchable, so this only happens when the # process is in an uninterruptible kernel wait (e.g. stuck - # on an NFS mount). Nothing more we can do — fall through - # to ``communicate()`` to drain whatever pipes are open. + # on an NFS mount). Nothing more we can do — the readers + # keep whatever output already arrived. + pass + else: + try: + proc.wait(timeout=_KILL_REAP_TIMEOUT_S) + except subprocess.TimeoutExpired: pass - # ``communicate()`` after a kill is safe and gathers any pending - # output that buffered before the signal landed. - try: - stdout, stderr = proc.communicate(timeout=_KILL_REAP_TIMEOUT_S) - except subprocess.TimeoutExpired as exc: - # A user command can intentionally detach a descendant without using - # ``run_in_background`` (for example ``nohup server ... &``). The - # shell exits, but the descendant's intermediary shell may retain our - # pipe, so communicate cannot observe EOF. TimeoutExpired still - # carries everything already read; preserve it instead of replacing a - # successful command's output with "(Bash completed with no output)". - def _captured(value: Any) -> str: - if isinstance(value, bytes): - return value.decode(errors="replace") - return value if isinstance(value, str) else "" - - stdout = _captured(exc.output) - stderr = _captured(exc.stderr) - for pipe in (proc.stdout, proc.stderr): - if pipe is not None: - try: - pipe.close() - except OSError: - pass + # Join the drain threads briefly. A detached descendant (``nohup + # server ... &``) can hold the pipe open past the shell's exit, so a + # reader may never see EOF — the old ``communicate(timeout=...)`` + # TimeoutExpired edge. Daemon threads make abandonment safe, and the + # chunk lists already hold everything read so far, which is exactly + # what the old path salvaged from ``TimeoutExpired``. + for r in readers: + r.join(timeout=_KILL_REAP_TIMEOUT_S) return _BashRunResult( returncode=proc.returncode if proc.returncode is not None else -1, - stdout=stdout or "", - stderr=stderr or "", + stdout=b"".join(out_chunks).decode(errors="replace"), + stderr=b"".join(err_chunks).decode(errors="replace"), interrupted=interrupted, timed_out=timed_out, + idle_timed_out=idle_timed_out, ) # ``\b`` is the WRONG boundary for a command NAME: ``-`` is a non-word @@ -731,13 +778,52 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: output={"error": str(exc)}, is_error=True, ) + # Stuck-command detection for nano's unbounded-timeout contract: + # a command that stays COMPLETELY silent for the idle window while + # still running is presumed stuck; noisy long builds never trip + # it. Stock mode keeps None — its hard timeout already bounds the + # damage. Small floor so a typo'd env cannot make it flap. + _idle_timeout_s: int | None = None + if _nano: + try: + _idle_timeout_s = max( + 5, int(_os_mod.environ.get("NANO_BASH_IDLE_TIMEOUT_S", "600")) + ) + except (ValueError, TypeError): + _idle_timeout_s = 600 + run_result = _run_bash_with_abort( argv, cwd=str(cwd), timeout_s=timeout_s, abort_signal=_get_abort_signal(context), + idle_timeout_s=_idle_timeout_s, ) + if run_result.idle_timed_out: + idle_msg = ( + f"Command produced no output for {_idle_timeout_s}s and was " + "killed as likely stuck (waiting for input it will never " + "receive, deadlocked, or hung). If it is interactive, re-run " + "it non-interactively (--yes/-y flags, `printf 'answer\\n' |`, " + "`Command was diff --git a/tests/nano/test_nano_bash.py b/tests/nano/test_nano_bash.py index 55dd2f99..3a46d8da 100644 --- a/tests/nano/test_nano_bash.py +++ b/tests/nano/test_nano_bash.py @@ -56,6 +56,62 @@ def test_minimum_floor_still_enforced_in_nano(ctx): _run(ctx, timeout=10) +def test_idle_watchdog_kills_silent_stuck_command(ctx, monkeypatch): + import time + + monkeypatch.setenv("NANO_BASH_IDLE_TIMEOUT_S", "5") + set_nano_mode(True) + t0 = time.monotonic() + result = BashTool.call({"command": "sleep 60"}, ctx) + elapsed = time.monotonic() - t0 + assert elapsed < 15, f"watchdog too slow: {elapsed:.0f}s" + assert result.is_error + assert "no output" in str(result.output) + assert "re-run" in str(result.output) + + +def test_idle_watchdog_spares_chatty_long_command(ctx, monkeypatch): + monkeypatch.setenv("NANO_BASH_IDLE_TIMEOUT_S", "5") + set_nano_mode(True) + result = BashTool.call( + {"command": "for i in 1 2 3 4; do echo tick-$i; sleep 2; done; echo chatty-done"}, + ctx, + ) + assert not result.is_error + assert "chatty-done" in str(result.output) + + +def test_pipe_drain_handles_output_beyond_pipe_buffer(ctx): + # >64KB written before exit used to fill the un-drained pipe and block + # the child forever (masked by the stock hard timeout as a bogus + # "timed out"). The concurrent readers drain it. + set_nano_mode(True) + result = BashTool.call( + {"command": "echo drained-ok; python3 -c \"print('x'*300000)\""}, ctx + ) + assert not result.is_error + assert "drained-ok" in str(result.output) + assert result.output.get("exit_code") == 0 + + +def test_pipe_drain_fixes_stock_mode_too(ctx): + result = BashTool.call( + {"command": "echo stock-ok; python3 -c \"print('y'*300000)\""}, ctx + ) + assert not result.is_error + assert "stock-ok" in str(result.output) + assert result.output.get("exit_code") == 0 + + +def test_stock_has_no_idle_watchdog(ctx, monkeypatch): + # Stock keeps its hard-timeout model: a 7s-silent command with a 15s + # timeout completes even with the env set (the watchdog is nano-only). + monkeypatch.setenv("NANO_BASH_IDLE_TIMEOUT_S", "5") + result = BashTool.call({"command": "sleep 7 && echo stock-quiet-ok", "timeout": 15000}, ctx) + assert not result.is_error + assert "stock-quiet-ok" in str(result.output) + + def test_nano_bash_schema_drops_background_trap(): # run_in_background needs TaskOutput (absent in nano) — a backgrounded # command strands its output; the nano schema must not advertise it. From 447446e30f0b8f9c84c1dde3f4b5921c065daa45 Mon Sep 17 00:00:00 2001 From: agentforce314 Date: Sun, 16 Aug 2026 18:35:22 -0700 Subject: [PATCH 5/5] feat(nano): pi-style rolling-tail truncation + full-output spill file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of pi's shell-output.ts capture model (the one residual bash-layer advantage pi held after round 3): * Tail-keep, not head-keep: for a long build the signal is at the END (the error, the final metrics); stock truncation keeps s[:limit] and drops exactly that. Nano replies now show the last BASH_MAX_OUTPUT_LENGTH chars. * The full output survives: once a stream crosses the limit it spills, complete from byte 0, to a temp file whose path is in the footer — [Showing last N of M lines (X KB total). Full output: /tmp/bash-...] — so the model greps the whole log instead of re-running the build. * Bounded memory: the drain keeps only ~2x limit per stream in a rolling deque (pi's lazy-lossless trigger: file created at 1x while memory still holds everything), so a multi-GB log no longer accumulates unbounded in chunk lists. Stock mode is byte-identical: the spool is wired only under the nano gate; all four result paths (normal/interrupted/timeout/idle) branch on the attached TailCapture and fall back to the old truncate_output. Nano Bash doc tells the model the tail+file contract (still <500 chars). Tests: 5 new (tail-keep+spill, small passthrough, stderr-separate, UTF-8 re-align on trim, stock unchanged); pipe-drain test marker moved to the end (tail-keep); nano suite 81 passed, stock bash suites 142 passed. Live smoke: 4001-line fake build log -> reply ends with the FATAL line + footer, spill file complete (169,826 bytes). Co-Authored-By: Claude Fable 5 --- src/nano/bash_tail.py | 185 ++++++++++++++++++++++++ src/nano/tool_docs.py | 4 +- src/tool_system/tools/bash/bash_tool.py | 134 ++++++++++++++--- tests/nano/test_nano_bash.py | 92 +++++++++++- 4 files changed, 391 insertions(+), 24 deletions(-) create mode 100644 src/nano/bash_tail.py diff --git a/src/nano/bash_tail.py b/src/nano/bash_tail.py new file mode 100644 index 00000000..ce8d7cd4 --- /dev/null +++ b/src/nano/bash_tail.py @@ -0,0 +1,185 @@ +"""pi-style rolling-tail output capture for nano bash. + +Port of pi's shell-output.ts capture model (packages/agent/src/harness/ +utils/shell-output.ts). Two properties, both absent from the stock tool: + +* **Tail-keep, not head-keep.** For a long build or training run the + signal is at the END of the output (the error, the final metrics); + stock truncation keeps ``s[:limit]`` and drops exactly that. +* **The full output survives.** Once a stream crosses the reply limit it + spills, complete, to a temp file whose path is handed to the model — + so the model can grep the whole log afterward instead of re-running. + +The lazy spill is lossless the way pi's is: memory is allowed to hold up +to 2x the limit before any trimming, and the spill file is created the +moment the total crosses 1x — everything captured so far is still in +memory at that point, so the file is complete from byte 0 and every later +chunk is appended as it arrives. Memory stays bounded at ~2x the limit +no matter how much the command prints. + +Wired only under nano (the bash tool's nano gate); stock bash keeps its +unbounded in-memory capture and head-keep truncation, byte-identical. +Each spool is normally fed by exactly one drain thread, but a detached +descendant holding the pipe open can leave that thread alive past the +reader join — ``finish()`` then races a late ``feed()`` — so the tiny +uncontended lock is load-bearing, not decoration. +""" + +from __future__ import annotations + +import tempfile +import threading +from collections import deque +from dataclasses import dataclass + + +@dataclass +class TailCapture: + """Finished capture of one stream (stdout or stderr).""" + + text: str # decoded rolling tail — the whole stream when small + total_bytes: int + total_lines: int + spilled: bool + # Path of the complete-output temp file; None when spilling was + # requested but failed (disk error) — the tail is then all we have. + spill_path: str | None + + +class TailSpool: + def __init__(self, limit_bytes: int, label: str) -> None: + self._limit = max(1, limit_bytes) + self._label = label + self._chunks: deque[bytes] = deque() + self._mem_bytes = 0 + self._total_bytes = 0 + self._newlines = 0 + self._ends_with_newline = True + self._spilled = False + self._spill_failed = False + self._file = None + self._path: str | None = None + self._lock = threading.Lock() + self._finished = False + + def feed(self, chunk: bytes) -> None: + if not chunk: + return + with self._lock: + self._feed_locked(chunk) + + def _feed_locked(self, chunk: bytes) -> None: + if self._finished: + # A straggler chunk from an abandoned drain thread after the + # result was already assembled — nothing left to attach it to. + return + self._total_bytes += len(chunk) + self._newlines += chunk.count(b"\n") + self._ends_with_newline = chunk.endswith(b"\n") + self._chunks.append(chunk) + self._mem_bytes += len(chunk) + + if not self._spilled and self._total_bytes > self._limit: + self._spilled = True + self._open_spill() + + if self._file is not None: + try: + self._file.write(chunk) + except (OSError, ValueError): + self._abandon_spill() + + # Trim only after the spill file exists (or failed) — before that + # the memory copy is the only complete copy. + if self._spilled: + while self._mem_bytes > 2 * self._limit and len(self._chunks) > 1: + self._mem_bytes -= len(self._chunks.popleft()) + + def _open_spill(self) -> None: + try: + self._file = tempfile.NamedTemporaryFile( + mode="wb", + prefix=f"bash-{self._label}-", + suffix=".log", + delete=False, + ) + self._path = self._file.name + # Nothing has been trimmed yet, so this is the stream from + # byte 0. The chunk that tripped the limit is already in + # ``_chunks`` and gets written by the caller's append. + for prior in list(self._chunks)[:-1]: + self._file.write(prior) + except OSError: + self._abandon_spill() + + def _abandon_spill(self) -> None: + self._spill_failed = True + if self._file is not None: + try: + self._file.close() + except OSError: + pass + self._file = None + self._path = None + + def finish(self) -> TailCapture: + with self._lock: + return self._finish_locked() + + def _finish_locked(self) -> TailCapture: + self._finished = True + if self._file is not None: + try: + self._file.close() + except OSError: + self._abandon_spill() + self._file = None + tail = b"".join(self._chunks) + if self._spilled: + # The front of the tail may sit mid-UTF-8-sequence after a + # trim; skip continuation bytes so the decode starts clean + # (pi's trimToLastUtf8Bytes). + start = 0 + while start < len(tail) and (tail[start] & 0xC0) == 0x80: + start += 1 + tail = tail[start:] + total_lines = self._newlines + ( + 1 if self._total_bytes and not self._ends_with_newline else 0 + ) + return TailCapture( + text=tail.decode(errors="replace"), + total_bytes=self._total_bytes, + total_lines=total_lines, + spilled=self._spilled, + spill_path=self._path, + ) + + +def _format_size(n: int) -> str: + if n < 1024: + return f"{n}B" + if n < 1024 * 1024: + return f"{n / 1024:.1f}KB" + return f"{n / (1024 * 1024):.1f}MB" + + +def render_tail(capture: TailCapture, limit_chars: int) -> str: + """Reply text for one stream: the last ``limit_chars`` plus a footer + naming the totals and the complete-output file (pi's bash.ts footer). + + Small streams (never spilled, within the limit) pass through + untouched — byte-identical to today's small-output replies. + """ + if not capture.spilled and len(capture.text) <= limit_chars: + return capture.text + shown = capture.text[-limit_chars:] + shown_lines = shown.count("\n") + (0 if shown.endswith("\n") else 1) + header = ( + f"[Showing last {shown_lines} of {capture.total_lines} lines " + f"({_format_size(capture.total_bytes)} total)." + ) + if capture.spill_path is not None: + footer = f"{header} Full output: {capture.spill_path}]" + else: + footer = f"{header} Earlier output was discarded.]" + return f"{shown}\n\n{footer}" diff --git a/src/nano/tool_docs.py b/src/nano/tool_docs.py index cf84d0f4..b0db0d81 100644 --- a/src/nano/tool_docs.py +++ b/src/nano/tool_docs.py @@ -27,7 +27,9 @@ "single call — pass timeout (ms) only when you want one. A " "command with NO output for 10 minutes is killed as stuck, so " "keep long jobs chatty (verbose flags, tee to a log). Long " - "output is truncated. Quote paths containing spaces." + "output returns only the tail plus the path of a file holding " + "the full output — grep it, don't re-run. Quote paths " + "containing spaces." ), # Edit is intentionally absent: the nano registry swaps in # NanoEditTool (src/nano/edit_tool.py), which carries its own doc for diff --git a/src/tool_system/tools/bash/bash_tool.py b/src/tool_system/tools/bash/bash_tool.py index b867bb15..ae0aa523 100644 --- a/src/tool_system/tools/bash/bash_tool.py +++ b/src/tool_system/tools/bash/bash_tool.py @@ -38,6 +38,12 @@ class _BashRunResult: # window while still running) — the "stuck forever" detector for # nano's unbounded-timeout contract. idle_timed_out: bool = False + # pi-style rolling-tail captures (``src.nano.bash_tail.TailCapture``), + # attached only on nano runs. When present, ``stdout``/``stderr`` + # hold the (memory-bounded) tail and the capture carries the totals + # and the complete-output spill path. + stdout_tail: Any | None = None + stderr_tail: Any | None = None def _get_abort_signal(context: ToolContext) -> Any: @@ -65,6 +71,7 @@ def _run_bash_with_abort( timeout_s: int, abort_signal: Any | None, idle_timeout_s: int | None = None, + tail_limit_bytes: int | None = None, ) -> _BashRunResult: """Run ``argv`` with abort + timeout + output-idle supervision. @@ -86,6 +93,12 @@ def _run_bash_with_abort( while still running is presumed stuck (waiting for input it will never get, deadlocked, or hung) and is killed with an actionable error, while a noisy 40-minute build never trips it. + + ``tail_limit_bytes`` (nano runs) switches capture to pi's rolling + tail: memory holds only the last ~2x limit of each stream, and the + FULL stream spills to a temp file once it crosses the limit — so a + multi-GB build log costs bounded memory and stays greppable on disk. + ``None`` (stock) keeps the unbounded in-memory capture unchanged. """ # ``stdin=DEVNULL`` matches TS ``Shell.ts`` (stdio[0] = 'pipe' with the @@ -123,14 +136,24 @@ def _run_bash_with_abort( err_chunks: list[bytes] = [] last_output = [_time_mod.monotonic()] - def _drain(stream: Any, chunks: list[bytes]) -> None: + spool_out = spool_err = None + if tail_limit_bytes is not None: + from src.nano.bash_tail import TailSpool + + spool_out = TailSpool(tail_limit_bytes, "stdout") + spool_err = TailSpool(tail_limit_bytes, "stderr") + + def _drain(stream: Any, chunks: list[bytes], spool: Any = None) -> None: try: fd = stream.fileno() while True: chunk = _os_mod.read(fd, 65536) if not chunk: break - chunks.append(chunk) + if spool is not None: + spool.feed(chunk) + else: + chunks.append(chunk) last_output[0] = _time_mod.monotonic() except (OSError, ValueError): pass @@ -141,8 +164,12 @@ def _drain(stream: Any, chunks: list[bytes]) -> None: pass readers = [ - _threading.Thread(target=_drain, args=(proc.stdout, out_chunks), daemon=True), - _threading.Thread(target=_drain, args=(proc.stderr, err_chunks), daemon=True), + _threading.Thread( + target=_drain, args=(proc.stdout, out_chunks, spool_out), daemon=True + ), + _threading.Thread( + target=_drain, args=(proc.stderr, err_chunks, spool_err), daemon=True + ), ] for r in readers: r.start() @@ -202,13 +229,25 @@ def _drain(stream: Any, chunks: list[bytes]) -> None: for r in readers: r.join(timeout=_KILL_REAP_TIMEOUT_S) + stdout_tail = spool_out.finish() if spool_out is not None else None + stderr_tail = spool_err.finish() if spool_err is not None else None return _BashRunResult( returncode=proc.returncode if proc.returncode is not None else -1, - stdout=b"".join(out_chunks).decode(errors="replace"), - stderr=b"".join(err_chunks).decode(errors="replace"), + stdout=( + stdout_tail.text + if stdout_tail is not None + else b"".join(out_chunks).decode(errors="replace") + ), + stderr=( + stderr_tail.text + if stderr_tail is not None + else b"".join(err_chunks).decode(errors="replace") + ), interrupted=interrupted, timed_out=timed_out, idle_timed_out=idle_timed_out, + stdout_tail=stdout_tail, + stderr_tail=stderr_tail, ) # ``\b`` is the WRONG boundary for a command NAME: ``-`` is a non-word @@ -256,10 +295,30 @@ def _bare_command(name: str) -> str: is_silent_command, ) from .sleep_detection import detect_blocked_sleep_pattern -from .utils import strip_empty_lines, strip_leading_blank_lines, truncate_output +from .utils import ( + get_max_output_length, + strip_empty_lines, + strip_leading_blank_lines, + truncate_output, +) BASH_TOOL_NAME = "Bash" + +def _render_stream(text: str, tail_capture: Any) -> str: + """Reply text for one captured stream. + + Stock: head-keep ``truncate_output``, byte-identical to before. Nano + runs attach a ``TailCapture`` and get pi's tail-keep render — the + last ``BASH_MAX_OUTPUT_LENGTH`` chars plus a footer naming the + totals and the complete-output spill file. + """ + if tail_capture is None: + return truncate_output(text) + from src.nano.bash_tail import render_tail + + return render_tail(tail_capture, get_max_output_length()) + TOOL_SUMMARY_MAX_LENGTH = 80 @@ -792,12 +851,17 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: except (ValueError, TypeError): _idle_timeout_s = 600 + # pi-style rolling-tail capture (nano): a long build's signal is + # at the END of its output; keep the tail for the reply, bound + # memory, and spill the complete stream to a temp file the model + # can grep afterward. Stock keeps unbounded capture + head-keep. run_result = _run_bash_with_abort( argv, cwd=str(cwd), timeout_s=timeout_s, abort_signal=_get_abort_signal(context), idle_timeout_s=_idle_timeout_s, + tail_limit_bytes=get_max_output_length() if _nano else None, ) if run_result.idle_timed_out: @@ -810,15 +874,28 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: "stretches, re-run it with progress output (verbose flags, " "`| tee log`) or pass an explicit `timeout`." ) + # The teaching message must survive whole, so under nano it + # is prepended AFTER the tail render (which is bounded); + # stock composes-then-truncates exactly as before. + if run_result.stderr_tail is not None: + idle_stderr = ( + idle_msg + + "\n" + + _render_stream(run_result.stderr or "", run_result.stderr_tail) + ).strip() + else: + idle_stderr = truncate_output( + (idle_msg + "\n" + (run_result.stderr or "")).strip() + ) return ToolResult( name=BASH_TOOL_NAME, output={ "cwd": str(cwd), "exit_code": 143, - "stdout": truncate_output(run_result.stdout or ""), - "stderr": truncate_output( - (idle_msg + "\n" + (run_result.stderr or "")).strip() + "stdout": _render_stream( + run_result.stdout or "", run_result.stdout_tail ), + "stderr": idle_stderr, "idle_timed_out": True, }, is_error=True, @@ -834,8 +911,12 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: output={ "cwd": str(cwd), "exit_code": -1, - "stdout": truncate_output(run_result.stdout or ""), - "stderr": truncate_output(run_result.stderr or ""), + "stdout": _render_stream( + run_result.stdout or "", run_result.stdout_tail + ), + "stderr": _render_stream( + run_result.stderr or "", run_result.stderr_tail + ), "interrupted": True, }, is_error=True, @@ -872,18 +953,29 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: timeout_marker = ( f"Command timed out after {format_duration(timeout_s * 1000)}" ) - stderr_with_marker = ( - f"{timeout_marker} {existing_stderr}" - if existing_stderr - else timeout_marker - ) + if run_result.stderr_tail is not None: + # Nano: render the stream first (tail + spill footer), + # then prepend the marker so it always survives. + rendered = _render_stream(existing_stderr, run_result.stderr_tail) + timeout_stderr = ( + f"{timeout_marker} {rendered}" if rendered else timeout_marker + ) + else: + stderr_with_marker = ( + f"{timeout_marker} {existing_stderr}" + if existing_stderr + else timeout_marker + ) + timeout_stderr = truncate_output(stderr_with_marker) return ToolResult( name=BASH_TOOL_NAME, output={ "cwd": str(cwd), "exit_code": 143, - "stdout": truncate_output(run_result.stdout or ""), - "stderr": truncate_output(stderr_with_marker), + "stdout": _render_stream( + run_result.stdout or "", run_result.stdout_tail + ), + "stderr": timeout_stderr, "timed_out": True, }, is_error=False, @@ -920,8 +1012,8 @@ def _bash_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: # roam freely but the UI cwd clamps to the workspace). pass - stdout = truncate_output(completed_stdout) - stderr = truncate_output(completed_stderr) + stdout = _render_stream(completed_stdout, run_result.stdout_tail) + stderr = _render_stream(completed_stderr, run_result.stderr_tail) interpretation = interpret_command_result( command, completed_returncode, completed_stdout, completed_stderr, diff --git a/tests/nano/test_nano_bash.py b/tests/nano/test_nano_bash.py index 3a46d8da..f60663e9 100644 --- a/tests/nano/test_nano_bash.py +++ b/tests/nano/test_nano_bash.py @@ -84,10 +84,11 @@ def test_idle_watchdog_spares_chatty_long_command(ctx, monkeypatch): def test_pipe_drain_handles_output_beyond_pipe_buffer(ctx): # >64KB written before exit used to fill the un-drained pipe and block # the child forever (masked by the stock hard timeout as a bogus - # "timed out"). The concurrent readers drain it. + # "timed out"). The concurrent readers drain it. Marker LAST: nano + # keeps the tail of long output (pi-style), not the head. set_nano_mode(True) result = BashTool.call( - {"command": "echo drained-ok; python3 -c \"print('x'*300000)\""}, ctx + {"command": "python3 -c \"print('x'*300000)\"; echo drained-ok"}, ctx ) assert not result.is_error assert "drained-ok" in str(result.output) @@ -112,6 +113,93 @@ def test_stock_has_no_idle_watchdog(ctx, monkeypatch): assert "stock-quiet-ok" in str(result.output) +def _spill_path_from(text: str) -> str: + import re + + m = re.search(r"Full output: (\S+)\]", text) + assert m, f"no spill footer in: {text[-400:]}" + return m.group(1) + + +def test_nano_tail_keep_with_full_output_spill(ctx, monkeypatch): + # pi parity: long output keeps the TAIL (where a build's error and + # final metrics live) and the complete stream spills to a temp file + # whose path the model gets — instead of stock's head-keep that + # drops exactly the interesting part and discards the rest forever. + monkeypatch.setenv("BASH_MAX_OUTPUT_LENGTH", "2000") + set_nano_mode(True) + cmd = ( + "echo HEAD-MARKER; " + "python3 -c \"print('filler-line\\n'*800, end='')\"; " + "echo TAIL-MARKER" + ) + result = BashTool.call({"command": cmd}, ctx) + assert not result.is_error + assert result.output.get("exit_code") == 0 + out = result.output["stdout"] + assert "TAIL-MARKER" in out + assert "HEAD-MARKER" not in out # tail-keep, not head-keep + assert "of 802 lines" in out # HEAD + 800 fillers + TAIL + assert len(out) < 2000 + 400 # bounded reply: limit + footer slack + + from pathlib import Path + + full = Path(_spill_path_from(out)).read_text() + assert full.startswith("HEAD-MARKER") # complete from byte 0 + assert full.endswith("TAIL-MARKER\n") + assert full.count("\n") == 802 + + +def test_nano_small_output_passes_through_unchanged(ctx, monkeypatch): + monkeypatch.setenv("BASH_MAX_OUTPUT_LENGTH", "2000") + set_nano_mode(True) + result = BashTool.call({"command": "echo small-ok"}, ctx) + out = result.output["stdout"] + assert "small-ok" in out + assert "[Showing" not in out and "Full output" not in out + + +def test_nano_stderr_spills_separately_from_stdout(ctx, monkeypatch): + monkeypatch.setenv("BASH_MAX_OUTPUT_LENGTH", "2000") + set_nano_mode(True) + cmd = ( + "python3 -c \"import sys; sys.stderr.write('err-line\\n'*600); " + "sys.stderr.write('ERR-TAIL\\n')\"; echo out-ok" + ) + result = BashTool.call({"command": cmd}, ctx) + assert "out-ok" in result.output["stdout"] + assert "Full output" not in result.output["stdout"] # stdout was small + err = result.output["stderr"] + assert "ERR-TAIL" in err + from pathlib import Path + + full = Path(_spill_path_from(err)).read_text() + assert full.startswith("err-line") and full.endswith("ERR-TAIL\n") + + +def test_nano_tail_decode_never_splits_multibyte(ctx, monkeypatch): + # The rolling-tail trim can cut mid-UTF-8-sequence; the render must + # re-align (pi's trimToLastUtf8Bytes) so no replacement chars leak. + monkeypatch.setenv("BASH_MAX_OUTPUT_LENGTH", "1000") + set_nano_mode(True) + result = BashTool.call({"command": "python3 -c \"print('é'*5000)\""}, ctx) + out = result.output["stdout"] + assert "Full output:" in out + assert "�" not in out + + +def test_stock_truncation_unchanged_head_keep(ctx, monkeypatch): + monkeypatch.setenv("BASH_MAX_OUTPUT_LENGTH", "2000") + result = BashTool.call( + {"command": "echo STOCK-HEAD; python3 -c \"print('line\\n'*1000, end='')\""}, + ctx, + ) + out = result.output["stdout"] + assert "STOCK-HEAD" in out # head kept + assert "lines truncated] ..." in out # stock marker format + assert "Full output:" not in out # no spill file in stock + + def test_nano_bash_schema_drops_background_trap(): # run_in_background needs TaskOutput (absent in nano) — a backgrounded # command strands its output; the nano schema must not advertise it.