diff --git a/docs/cookbook/07-slack.md b/docs/cookbook/07-slack.md index 3b8705a..c2380dc 100644 --- a/docs/cookbook/07-slack.md +++ b/docs/cookbook/07-slack.md @@ -25,6 +25,7 @@ an afterthought. The defaults: | `demo`, `run`, `plan`, `models`, `replay`, `diff`, `trace`, `metrics`, `viz` | `serve` | | Paths that resolve inside the bot's working directory | Any path that escapes it (`trace ../../.env` is refused before a process spawns) | | The budget, policy and trace flags each command already has | `--registry` (imports an arbitrary module), `--config`, `--json`, `--no-color` | +| `plan --registry`, for exactly the two registries the package ships | any other `--registry` value | | `agent`, only behind the double opt-in below | `--model` / `--reviewer-model`, unless the operator opts in | With `--model` off, every reachable command runs the scripted, spend-free @@ -114,6 +115,24 @@ upward-directory search that the model gateway performs is deliberately not used here: a bot that a whole workspace can drive must not discover credentials in a file the operator did not point it at. +## A `plan` that reads + +The default planning registry is the incident-response demo: its node bodies +are stubs, so a goal like "summarise the docs here" gets an honest negative +(or, if phrased vaguely enough, a hollow success). The shipped alternative +has bodies that really read — `survey` / `read` / `summarise`, read-only, +confined to the bot's working directory: + +``` +@grapharc plan "summarise the docs in this workspace" --registry grapharc.examples.plan_docs:build_registry --trace docs.jsonl --run-id docs-1 +``` + +Works with the scripted planner (free) and with `--model`; either way the +notes in the final state carry actual file names, titles and excerpts, +because the reading is operator code, not model output. `--registry` from +Slack accepts exactly these two shipped modules and nothing else — the flag's +general form imports arbitrary code, which stays refused. + ## The `agent` opt-in `agent` is the command with file tools, which is exactly why it is off by diff --git a/grapharc/cli/plan.py b/grapharc/cli/plan.py index d70edb7..1cb2735 100644 --- a/grapharc/cli/plan.py +++ b/grapharc/cli/plan.py @@ -146,13 +146,29 @@ def resolve_edge_policy(policy_path: Path | None, *, tenant: str) -> tuple[Any, return policy, f"{policy_path} (tenant {tenant!r}, {len(policy.rules)} edge rule(s))" -def _model_for(spec: str | None) -> tuple[Any, str]: - """The scripted planner by default; a real backend when asked for one.""" +def _model_for(spec: str | None, registry_target: str = DEFAULT_REGISTRY) -> tuple[Any, str]: + """The scripted planner by default; a real backend when asked for one. + + The scripted replies come from the registry module when it supplies + `scripted_planner_replies`, because a script that proposes one registry's + kinds against another registry's catalog is rejected every round — the + incident replies against the docs registry produced five rounds of + `unregistered_node` and a `planning_failed`. The incident module's replies + stay the fallback for modules that ship none. + """ if spec is None: - from grapharc.examples.plan_incident import scripted_planner_replies from grapharc.testing import ScriptedChatModel - return ScriptedChatModel(responses=scripted_planner_replies()), "scripted" + module_name = registry_target.split(":", 1)[0] + try: + module = importlib.import_module(module_name) + except ImportError as exc: + raise PlanSetupError(f"--registry {registry_target!r}: {exc}") from exc + replies = getattr(module, "scripted_planner_replies", None) + if replies is None: + from grapharc.examples.plan_incident import scripted_planner_replies as replies + + return ScriptedChatModel(responses=replies()), "scripted" from grapharc.gateway import get_model return get_model(spec), spec @@ -191,7 +207,7 @@ def plan( tenant = settings.resolve("tenant", tenant, "default") max_rounds = settings.resolve("max_rounds", max_rounds, 8) max_tokens = settings.resolve("max_tokens", max_tokens, 100_000) - model, model_description = _model_for(model_spec) + model, model_description = _model_for(model_spec, registry_target) bundle = resolve_registry(registry_target, model) registry, state_schema, writes = bundle.registry, bundle.state_schema, bundle.writes edge_policy, policy_description, policy_source = resolve_or_generate_policy( diff --git a/grapharc/examples/plan_docs.py b/grapharc/examples/plan_docs.py new file mode 100644 index 0000000..76f533a --- /dev/null +++ b/grapharc/examples/plan_docs.py @@ -0,0 +1,184 @@ +"""A planning registry whose kinds can actually read — ROADMAP §12.1's sequel. + +`plan_incident` proves the governance; its node bodies are stubs, and a goal +like "summarise the docs in this workspace" gets either an honest negative or +a hollow success, depending on how vague the goal is. This registry closes +that gap for one bounded job: **reading documentation under the current +working directory and reporting what is there.** Three kinds: + + survey list the documentation files under cwd + read read them and record an excerpt of each + summarise one note distilling title + first paragraph per file + +Everything is deterministic operator code. The model still does the +*planning* — which kinds, in what order, replanning on refusal — but no node +body contains a model call, so a run with a scripted planner produces real +file content and a run with `--model` spends tokens only on rounds. + +The confinement matters more than the capability: bodies read, never write, +and only under `Path.cwd()` — which, launched from the Slack bot, is the +bot's working directory. A proposal cannot steer them elsewhere because a +proposal names kinds and carries no arguments; there is no path field to +inject (the gap issue #10 tracks does not open here because these bodies +take no arguments at all). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pydantic import BaseModel + +from grapharc.harness.permissions import Decision +from grapharc.planner import CostEstimate, EdgePolicy, EdgeRule, NodeRegistry, NodeSpec + +#: Read at most this many files, this much of each. Documentation, not a dump. +MAX_FILES = 20 +MAX_CHARS = 40_000 +_SUFFIXES = (".md", ".txt", ".rst") +_SKIP_DIRS = {".git", ".grapharc", ".venv", "__pycache__", "node_modules"} + + +class DocsState(BaseModel): + """`notes` is deliberately the whole record: the loop's goal check reads it.""" + + goal: str = "" + notes: list[str] = [] + + +def _docs_files(root: Path) -> list[Path]: + """Every documentation file under `root`, and nothing outside it.""" + found = [] + for path in sorted(root.rglob("*")): + if len(found) >= MAX_FILES: + break + if not path.is_file() or path.suffix.lower() not in _SUFFIXES: + continue + if any(part in _SKIP_DIRS for part in path.parts): + continue + # Belt and braces: rglob cannot leave root, but a symlink can point + # anywhere. Resolve and check before a single byte is read. + if not path.resolve().is_relative_to(root): + continue + found.append(path) + return found + + +def _excerpt(path: Path, root: Path) -> str: + text = path.read_text(encoding="utf-8", errors="replace")[:MAX_CHARS] + lines = [line.strip() for line in text.splitlines()] + title = next((line.lstrip("# ") for line in lines if line), path.name) + body = next((line for line in lines if line and not line.startswith("#")), "") + return f"{path.relative_to(root)}: {title}" + (f" — {body[:160]}" if body else "") + + +def _survey_body(state: DocsState) -> dict: + root = Path.cwd().resolve() + files = _docs_files(root) + listing = ", ".join(str(f.relative_to(root)) for f in files) or "none found" + return {"notes": [*state.notes, f"survey: {len(files)} documentation file(s): {listing}"]} + + +def _read_body(state: DocsState) -> dict: + root = Path.cwd().resolve() + notes = [f"read {f.relative_to(root)}: {_excerpt(f, root)}" for f in _docs_files(root)] + return {"notes": [*state.notes, *(notes or ["read: nothing to read"])]} + + +def _summarise_body(state: DocsState) -> dict: + root = Path.cwd().resolve() + files = _docs_files(root) + if not files: + summary = "summary: no documentation files under the working directory" + else: + parts = "; ".join(_excerpt(f, root) for f in files[:10]) + summary = f"summary of {len(files)} file(s): {parts}" + return {"notes": [*state.notes, summary]} + + +_BODIES = {"survey": _survey_body, "read": _read_body, "summarise": _summarise_body} + + +def _factory(build: Any) -> Any: + # `build` is the materialiser's NodeBuild: `name` is the instance the + # planner chose ("readme_survey"), `kind` is what the registry licensed. + # Behaviour keys on the kind; the name is the planner's business. + body = _BODIES[build.kind] + body.writes = {"notes"} + return body + + +WRITES: dict[str, set[str]] = {kind: {"notes"} for kind in _BODIES} + + +def build_registry() -> NodeRegistry: + """Three read-only kinds. Absence is refusal; there is no write kind to deny.""" + return NodeRegistry( + [ + NodeSpec( + name="survey", + description="list the documentation files under the working directory", + factory=_factory, + worst_case=CostEstimate(iterations=1, tokens=300), + ), + NodeSpec( + name="read", + description="read each documentation file and record an excerpt", + factory=_factory, + worst_case=CostEstimate(iterations=1, tokens=1500), + ), + NodeSpec( + name="summarise", + description="distil what was read into one summary note", + factory=_factory, + worst_case=CostEstimate(iterations=1, tokens=800), + ), + ] + ) + + +def default_edge_policy() -> EdgePolicy: + """Allow every transition: nothing here mutates, so nothing needs denying.""" + return EdgePolicy(rules=(EdgeRule(action=Decision.ALLOW),)) + + +def scripted_planner_replies() -> list[str]: + """One reply: survey → read → summarise. Read by `grapharc plan` when no + `--model` is given, so the free path exercises the same registry the paid + one does. In an empty directory the chain still yields three notes, so the + loop's goal check is satisfied either way.""" + import json + + from grapharc.runtime.graph import END, START + + chain = ["survey", "read", "summarise"] + endpoints = [START, *chain, END] + return [ + json.dumps( + { + "nodes": [{"name": kind} for kind in chain], + "edges": [ + {"source": a, "target": b} + for a, b in zip(endpoints, endpoints[1:], strict=False) + ], + } + ) + ] + + +STATE_SCHEMA = DocsState + +#: Nothing in this registry writes outside run state, so the policy generator +#: has nothing to deny — and saying so explicitly beats being defaulted. +MUTATING_KINDS: tuple[str, ...] = () + +__all__ = [ + "MUTATING_KINDS", + "STATE_SCHEMA", + "WRITES", + "DocsState", + "build_registry", + "default_edge_policy", + "scripted_planner_replies", +] diff --git a/grapharc/slack/command.py b/grapharc/slack/command.py index e23c152..9341d84 100644 --- a/grapharc/slack/command.py +++ b/grapharc/slack/command.py @@ -13,7 +13,9 @@ - **Flags are allowlisted per subcommand.** `--registry MODULE:ATTR` imports an arbitrary module on the host, `--config PATH` swaps the governing file, and `--json`/`--no-color` fight the bot's own output handling — none are - reachable from Slack. + reachable from Slack, with one carve-out: `plan --registry` accepts exactly + the registry modules this package ships (`PLAN_REGISTRIES`), because code + the wheel itself carries is the operator's, not the requester's. - **`--model` is refused unless the operator opted in**, because it reaches a paid backend. Without it every allowed command runs the scripted, spend-free path; the default answer to "can Slack cost me money?" is no. @@ -47,11 +49,25 @@ class CommandSpec: path_positionals: frozenset[int] = frozenset() # value flags that reach a paid backend; admitted only with allow_model model_flags: frozenset[str] = frozenset() + # flag -> the exact values it may take. How `--registry` stays shut against + # arbitrary imports while the registries this package ships stay reachable. + choice_flags: dict[str, frozenset[str]] = field(default_factory=dict) _BUDGET = {"--max-tokens": False, "--max-iterations": False, "--max-seconds": False} _NAMED_RUN = {"--trace": True, "--run-id": False} +#: The only `--registry` values `plan` accepts from Slack: the two registries +#: this package ships. The flag stays refused everywhere else — its value is +#: an arbitrary `module:attr` import, which is exactly what the gate exists to +#: prevent — but a registry the wheel itself carries is the operator's code. +PLAN_REGISTRIES = frozenset( + { + "grapharc.examples.plan_incident:build_registry", + "grapharc.examples.plan_docs:build_registry", + } +) + ALLOWED_COMMANDS: dict[str, CommandSpec] = { "demo": CommandSpec( value_flags={"--trace": True, "--memory": True, "--memory-backend": False}, @@ -77,6 +93,7 @@ class CommandSpec: "--max-tokens": False, }, model_flags=frozenset({"--model"}), + choice_flags={"--registry": PLAN_REGISTRIES}, ), "models": CommandSpec(bool_flags=frozenset({"--check"})), "agent": CommandSpec( @@ -188,8 +205,8 @@ def parse_command( "the operator enables it with GRAPHARC_SLACK_ALLOW_MODEL=1" ) is_path = False - elif flag in spec.value_flags: - is_path = spec.value_flags[flag] + elif flag in spec.choice_flags or flag in spec.value_flags: + is_path = spec.value_flags.get(flag, False) else: raise SlackCommandError(f"`{flag}` is not allowed on `{name}` from Slack") if eq: @@ -200,6 +217,11 @@ def parse_command( raise SlackCommandError(f"`{flag}` needs a value") value = rest[index + 1] index += 2 + if flag in spec.choice_flags and value not in spec.choice_flags[flag]: + allowed = ", ".join(f"`{v}`" for v in sorted(spec.choice_flags[flag])) + raise SlackCommandError( + f"`{flag}` accepts only the shipped registries from Slack: {allowed}" + ) if is_path: _confined(value, workdir) argv.extend([flag, value]) diff --git a/tests/test_plan_docs.py b/tests/test_plan_docs.py new file mode 100644 index 0000000..7ca9409 --- /dev/null +++ b/tests/test_plan_docs.py @@ -0,0 +1,111 @@ +"""The docs registry: node bodies that really read, confined to the cwd. + +`plan_incident` proves governance with stub bodies; `plan_docs` is the +shipped answer to "summarise the docs in this workspace" being either a +hollow success or an honest negative. These tests pin the three properties +that make it safe to expose through the Slack gate: bodies read real content, +they read only under the working directory, and the full governed loop +reaches `goal_met` with substantive notes rather than "x ran" markers. +""" + +from __future__ import annotations + +import json + +import pytest + +from grapharc.examples import plan_docs +from grapharc.examples.plan_incident import build_loop +from grapharc.runtime.graph import END, START +from grapharc.testing import ScriptedChatModel + + +@pytest.fixture() +def docs_dir(tmp_path, monkeypatch): + (tmp_path / "README.md").write_text("# Widget\n\nA widget that frobs.\n") + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "guide.md").write_text("# Guide\n\nStart by frobbing.\n") + (tmp_path / "code.py").write_text("print('not documentation')\n") + monkeypatch.chdir(tmp_path) + return tmp_path + + +def test_survey_lists_only_documentation_under_cwd(docs_dir): + notes = plan_docs._survey_body(plan_docs.DocsState())["notes"] + assert len(notes) == 1 + assert "2 documentation file(s)" in notes[0] + assert "README.md" in notes[0] + assert "code.py" not in notes[0] + + +def test_read_produces_real_content_not_ran_markers(docs_dir): + notes = plan_docs._read_body(plan_docs.DocsState())["notes"] + assert any("A widget that frobs." in note for note in notes) + assert not any(note.endswith(" ran") for note in notes) + + +def test_a_symlink_pointing_outside_the_cwd_is_never_read(docs_dir, tmp_path_factory): + outside = tmp_path_factory.mktemp("outside") / "secret.md" + outside.write_text("# Secret\n\nnot yours\n") + (docs_dir / "link.md").symlink_to(outside) + notes = plan_docs._read_body(plan_docs.DocsState())["notes"] + assert not any("not yours" in note for note in notes) + + +def test_the_cli_scripted_path_uses_this_registrys_replies(docs_dir, capsys): + """No `--model`, docs registry: the free path must reach the goal. + + Fails without `_model_for` reading `scripted_planner_replies` off the + registry module — the incident replies propose kinds this registry does + not have, and every round dies `unregistered_node`. + """ + from grapharc.cli.main import main + + code = main( + [ + "plan", + "summarise the docs", + "--registry", + "grapharc.examples.plan_docs:build_registry", + "--trace", + str(docs_dir / "t.jsonl"), + ] + ) + printed = capsys.readouterr().out + assert code == 0 + assert "goal_met" in printed + assert "A widget that frobs." in printed + + +def test_the_governed_loop_reaches_goal_met_with_substantive_notes(docs_dir, tmp_path): + # Instance names differ from kinds on purpose: a real planner invents + # names ("docs_survey" of kind "survey"), and behaviour must key on the + # kind — this is the shape that catches a factory keyed on the name. + chain = ["survey", "read", "summarise"] + names = [f"docs_{kind}" for kind in chain] + endpoints = [START, *names, END] + reply = json.dumps( + { + "nodes": [ + {"name": name, "kind": kind} + for name, kind in zip(names, chain, strict=True) + ], + "edges": [ + {"source": a, "target": b} + for a, b in zip(endpoints, endpoints[1:], strict=False) + ], + } + ) + loop = build_loop( + ScriptedChatModel(responses=[reply]), + registry=plan_docs.build_registry(), + state_schema=plan_docs.DocsState, + writes=plan_docs.WRITES, + edge_policy=plan_docs.default_edge_policy(), + ) + result = loop.run( + "summarise the docs", plan_docs.DocsState(goal="summarise the docs") + ) + assert result.succeeded + assert any("summary of 2 file(s)" in note for note in result.state.notes) + assert any("Widget" in note for note in result.state.notes) diff --git a/tests/test_slack_gateway.py b/tests/test_slack_gateway.py index 04dc4e7..23843f0 100644 --- a/tests/test_slack_gateway.py +++ b/tests/test_slack_gateway.py @@ -102,6 +102,20 @@ def test_registry_config_and_json_are_refused(tmp_path): parse_command(f"run graph.toml {flag}", workdir=tmp_path) +def test_plan_registry_admits_only_the_shipped_modules(tmp_path): + argv = parse_command( + "plan goal --registry grapharc.examples.plan_docs:build_registry", + workdir=tmp_path, + ) + assert argv[-1] == "grapharc.examples.plan_docs:build_registry" + with pytest.raises(SlackCommandError, match="shipped registries"): + parse_command("plan goal --registry os:system", workdir=tmp_path) + with pytest.raises(SlackCommandError, match="shipped registries"): + parse_command( + "plan goal --registry=evil.module:build_registry", workdir=tmp_path + ) + + def test_model_is_refused_by_default_and_admitted_on_opt_in(tmp_path): with pytest.raises(SlackCommandError, match="paid backend"): parse_command("plan 'a goal' --model mock/x", workdir=tmp_path)