diff --git a/CHANGELOG.md b/CHANGELOG.md index b6930b4e7..df41c0143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## Unreleased + +- Feature: the `kimi` backend now sends an explicit `reasoning_effort` (default `max`, overridable via `GRAPHIFY_KIMI_EFFORT`) for Kimi models that support it — K3 advertises `valid_efforts ["low","high","max"]` on `/models`, and sending nothing let the server default (`"high"` for K3) apply silently while the gemini backend already carried an effort setting. Forwarded by the existing request plumbing; no behaviour change for models that ignore the field. +- Feature: a new `openai-cli` backend (`--backend openai-cli`) runs semantic extraction and community labeling through the locally authenticated OpenAI Codex CLI (`codex exec`), so the work rides a ChatGPT OAuth subscription instead of a metered `OPENAI_API_KEY` — removing it would silently fall extraction back to metered API spend. The prompt travels via stdin (Linux caps a single argv entry at 128 KB; real chunks reach 240–306 KB), the sandbox is read-only so an agentic Codex cannot write to the corpus it reads, every configured MCP server is disabled per call through Codex's own per-server `enabled` override (a blanket `mcp_servers={}` deep-merges away; the measured cost was four ~152 MB servers per `codex exec`), token usage is read from the last `turn.completed` JSONL event without double-counting cached input, an empty-but-valid graph raises instead of triggering the hollow-response bisect (which once burned 87% of an hourly quota), and calls are forced serial unless `GRAPHIFY_OPENAI_CLI_PARALLEL=1`. `GRAPHIFY_OPENAI_CLI_MODEL` (default `gpt-5.6-sol`) and `GRAPHIFY_OPENAI_CLI_EFFORT` (default `ultra`) configure it; the credential gate accepts a present `codex` CLI in place of an API key, mirroring `claude-cli`. +- Feature: `graphify extract` now holds the same per-repo rebuild lock the watcher and git-hook rebuilds take, for its whole pipeline, so two extracts (or an extract racing `graphify update`) on one `graphify-out/` serialize instead of interleaving cache saves and clobbering `graph.json`; a contended run names the holder's PID, waits up to `GRAPHIFY_LOCK_TIMEOUT` seconds (default 600), and exits with an error instead of hanging forever behind a wedged rebuild. The lock file's PID payload and unlink-on-release contracts are unchanged. +- Feature: `graphify extract --fallback-backend ` (or `GRAPHIFY_FALLBACK_BACKEND`; the flag wins) retries the semantic pass once on a second backend when every chunk fails on the primary, so a missing SDK package, a bad key, or an outage no longer costs the whole build; the retry covers exactly the still-uncached files, `--model` stays with the primary backend (the fallback runs on its own default model), a typo'd fallback name is rejected before any API spend, and only a zero-success retry keeps the all-chunks-failed exit 1. +- Feature: `graphify watch --semantic` runs LLM-backed semantic extraction automatically when doc/paper/image files change, instead of only writing the `needs_update` flag; the extract runs as a subprocess so it serializes on the per-repo rebuild lock (in-process re-entry would self-deadlock on `flock`), a failed extract still falls back to the flag + `/graphify --update` instruction, and a successful run clears the flag so no stale prompt is left behind. `--backend`/`--fallback-backend` are forwarded to the extract and rejected without `--semantic` rather than being a silent no-op. +- Feature: `graphify export neo4j|falkordb --push` now sends nodes and edges in UNWIND batches (`--batch-size`, default 100) instead of one query per entry, so a remote push stops spending nearly all its time on round trips (~100x fewer for the default); rows are grouped by sanitized node label / relationship type first (those are baked into the Cypher text and cannot be parameters), the row payloads are exactly the old per-entry params, and UNWIND processes rows in order, so the MERGE/SET upsert semantics — including idempotent re-runs — are unchanged. + ## 0.9.48 (2026-08-20) - Fix: a control character in a node label or id no longer aborts the whole export; the GraphML and Obsidian exporters scrub only the characters those formats forbid (tab, newline, and non-ASCII letters are preserved), and `graph.json` and its byte-identity round-trip are untouched (#2897, thanks @abhay-codes07). diff --git a/graphify/__main__.py b/graphify/__main__.py index 155501a98..c39c8116d 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -534,6 +534,9 @@ def _run_cli() -> None: print(" --contributor \"Name\" tag who added it to the corpus") print(" --dir target directory (default: ./raw)") print(" watch watch a folder and rebuild the graph on code changes") + print(" --semantic also run LLM extraction automatically on doc/image changes") + print(" --backend extraction backend for --semantic") + print(" --fallback-backend fallback backend for --semantic") print(" update re-extract code files and update the graph (no LLM needed)") print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") @@ -544,13 +547,14 @@ def _run_cli() -> None: print(" --no-label keep 'Community N' placeholders (skip LLM community naming)") print(" --backend= backend to use for community naming (default: auto-detect)") print(" --model= model to use for community naming") - print(" --max-concurrency=N parallel community-labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") + # PATCHED FOR TELB-COCKPIT: openai-cli is also forced serial in llm.py. + print(" --max-concurrency=N parallel community-labeling LLM calls (default 4; forced to 1 for ollama/claude-cli/openai-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") print(" label (re)name communities with the configured LLM backend, regenerate report") print(" --missing-only keep existing labels and only name missing/placeholder communities") print(" --backend= backend to use (default: auto-detect from API keys)") print(" --model= model to use for community naming") - print(" --max-concurrency=N parallel labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") + print(" --max-concurrency=N parallel labeling LLM calls (default 4; forced to 1 for ollama/claude-cli/openai-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") print(" query \"\" BFS traversal of graph.json for a question") print(" --dfs use depth-first instead of breadth-first") diff --git a/graphify/cli.py b/graphify/cli.py index 02e55b944..050800f1f 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1730,14 +1730,57 @@ def dispatch_command(cmd: str) -> None: sys.exit(1) elif cmd == "watch": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + watch_semantic = False + watch_backend: str | None = None + watch_fallback_backend: str | None = None + watch_arg: str | None = None + args = sys.argv[2:] + i = 0 + while i < len(args): + a = args[i] + if a == "--semantic": + watch_semantic = True; i += 1 + elif a == "--backend" and i + 1 < len(args): + watch_backend = args[i + 1]; i += 2 + elif a.startswith("--backend="): + watch_backend = a.split("=", 1)[1]; i += 1 + elif a == "--fallback-backend" and i + 1 < len(args): + watch_fallback_backend = args[i + 1]; i += 2 + elif a.startswith("--fallback-backend="): + watch_fallback_backend = a.split("=", 1)[1]; i += 1 + elif a.startswith("-"): + print(f"error: unknown watch option: {a}", file=sys.stderr) + sys.exit(2) + else: + if watch_arg is not None: + print("error: watch accepts at most one path argument", file=sys.stderr) + sys.exit(2) + watch_arg = a; i += 1 + + if (watch_backend or watch_fallback_backend) and not watch_semantic: + # Without --semantic the watcher never runs an extract, so a + # backend flag would be a silent no-op the user believes took + # effect — reject it loudly instead. + print( + "error: --backend/--fallback-backend require --semantic " + "(they configure the automatic semantic extraction pass)", + file=sys.stderr, + ) + sys.exit(2) + + watch_path = Path(watch_arg) if watch_arg is not None else Path(".") if not watch_path.exists(): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) from graphify.watch import watch as _watch try: - _watch(watch_path) + _watch( + watch_path, + semantic=watch_semantic, + backend=watch_backend, + fallback_backend=watch_fallback_backend, + ) except ImportError as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) @@ -2509,9 +2552,9 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) print(" graphml [--graph PATH]", file=sys.stderr) - print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P] [--batch-size N]", file=sys.stderr) print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P] [--batch-size N]", file=sys.stderr) print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) sys.exit(1) @@ -2546,6 +2589,9 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" else os.environ.get("NEO4J_PASSWORD") ) or None + # UNWIND rows per round trip for the push sinks; the per-entry queries + # made a remote push spend nearly all its time on round trips. + push_batch_size = 100 i = 0 while i < len(args): a = args[i] @@ -2601,6 +2647,16 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": push_user = args[i + 1]; i += 2 elif a == "--password" and i + 1 < len(args): push_password = args[i + 1]; i += 2 + elif a == "--batch-size" and i + 1 < len(args): + try: + push_batch_size = int(args[i + 1]) + except ValueError: + print("error: --batch-size must be an integer", file=sys.stderr) + sys.exit(2) + if push_batch_size < 1: + print("error: --batch-size must be a positive integer", file=sys.stderr) + sys.exit(2) + i += 2 elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: candidate = Path(a) if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": @@ -2784,7 +2840,8 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print("error: --password required for --push", file=sys.stderr) sys.exit(1) result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) + password=push_password, communities=communities, + batch_size=push_batch_size) print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") else: from graphify.export import to_cypher as _to_cypher @@ -2795,7 +2852,8 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": if push_uri: from graphify.export import push_to_falkordb as _push result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) + password=push_password, communities=communities, + batch_size=push_batch_size) print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") else: from graphify.export import to_cypher as _to_cypher @@ -2888,6 +2946,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": if len(sys.argv) < 3: print( "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "[--fallback-backend B] " "[--model M] [--mode deep] [--out DIR|--output DIR] [--google-workspace] [--no-cluster] " "[--no-gitignore] [--code-only] [--no-dedup] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " @@ -2940,6 +2999,12 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": # --force parity with `graphify update`: the flag or GRAPHIFY_FORCE=1 # disables the incremental gate and skips semantic-cache reads (#1894). force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + # --fallback-backend: a second backend to retry the semantic pass on + # when EVERY chunk fails on the primary (missing SDK, bad key, an + # outage). The CLI flag wins over GRAPHIFY_FALLBACK_BACKEND. + fallback_backend: str | None = ( + os.environ.get("GRAPHIFY_FALLBACK_BACKEND", "").strip() or None + ) def _parse_int(name: str, raw: str) -> int: try: @@ -2971,6 +3036,10 @@ def _parse_float(name: str, raw: str) -> float: backend = args[i + 1]; i += 2 elif a.startswith("--backend="): backend = a.split("=", 1)[1]; i += 1 + elif a == "--fallback-backend" and i + 1 < len(args): + fallback_backend = args[i + 1]; i += 2 + elif a.startswith("--fallback-backend="): + fallback_backend = a.split("=", 1)[1]; i += 1 elif a == "--model" and i + 1 < len(args): model = args[i + 1]; i += 2 elif a.startswith("--model="): @@ -3088,849 +3157,1092 @@ def _parse_float(name: str, raw: str) -> float: # Persist corpus-shaping options so later update/watch/hook rebuilds # use the same file set as the initial extraction (#1886). from graphify.watch import ( + _rebuild_lock, _write_build_config as _write_build_cfg, _read_build_excludes as _read_build_ex, _read_build_gitignore as _read_build_gi, ) - # #1971 persistence: an explicit --no-gitignore persists False; a later - # flag-less `graphify extract` must NOT clobber it back to True, which - # would make the git-ignored code silently disappear again (the exact - # complaint #1971 is about). Honor the persisted value for THIS run when - # the flag is absent (read before the write below), and write False only - # when the flag is set — None leaves the setting as-is, mirroring how - # #1886 persists --exclude. - _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) - # An explicit list replaces the persisted one; omission reuses it. - _effective_excludes = cli_excludes or _read_build_ex(graphify_out) - _write_build_cfg( - graphify_out, - excludes=cli_excludes or None, - gitignore=False if no_gitignore else None, - ) + # Hold the per-repo rebuild lock for the entire pipeline. extract and + # the watcher/hook rebuilds share graphify-out/: two writers racing on + # the same output dir interleave cache saves and clobber graph.json, + # so a second run must wait its turn behind the same advisory flock + # _rebuild_code takes. Acquired exactly once here and never re-entered + # below — flock is per-open-file-description, so a second acquisition + # in this same process would deadlock against ourselves. A plain + # ``with`` (via ExitStack, to allow the escalate-to-blocking retry) + # keeps the unlink-on-release contract on every exit path: SystemExit + # from the sys.exit() calls below still unwinds the context manager. + import contextlib + try: + _lock_timeout = float(os.environ.get("GRAPHIFY_LOCK_TIMEOUT", "") or 600) + except ValueError: + _lock_timeout = 600.0 + with contextlib.ExitStack() as _lock_stack: + if not _lock_stack.enter_context(_rebuild_lock(graphify_out, blocking=False)): + # Contended: tell the user who holds it, then wait bounded so a + # wedged rebuild cannot hang a headless CI run forever. + try: + _holder = (graphify_out / ".rebuild.lock").read_text(encoding="utf-8").strip() + except OSError: + _holder = "" + _who = f" (pid {_holder})" if _holder else "" + print( + f"[graphify extract] waiting for another rebuild{_who} " + f"to finish (up to {int(_lock_timeout)}s, GRAPHIFY_LOCK_TIMEOUT to change)..." + ) + if not _lock_stack.enter_context( + _rebuild_lock(graphify_out, blocking=True, timeout=_lock_timeout) + ): + print( + f"error: gave up waiting for the rebuild lock after {int(_lock_timeout)}s; " + f"another graphify run is still holding {graphify_out / '.rebuild.lock'}", + file=sys.stderr, + ) + sys.exit(1) + # #1971 persistence: an explicit --no-gitignore persists False; a later + # flag-less `graphify extract` must NOT clobber it back to True, which + # would make the git-ignored code silently disappear again (the exact + # complaint #1971 is about). Honor the persisted value for THIS run when + # the flag is absent (read before the write below), and write False only + # when the flag is set — None leaves the setting as-is, mirroring how + # #1886 persists --exclude. + _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) + # An explicit list replaces the persisted one; omission reuses it. + _effective_excludes = cli_excludes or _read_build_ex(graphify_out) + _write_build_cfg( + graphify_out, + excludes=cli_excludes or None, + gitignore=False if no_gitignore else None, + ) - stages = _StageTimer(cli_timing) + stages = _StageTimer(cli_timing) - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - # #1925: a missing manifest.json must not degrade to a full scan that - # discards the existing graph's semantic layer. An existing graph.json - # is a sufficient incremental baseline: detect_incremental treats an - # absent manifest as "everything is new" (re-extract all, nothing - # deleted), and build_merge + _stale_graph_sources reconcile replaced - # and genuinely-deleted sources against the current corpus, so doc/ - # paper/image nodes survive a --code-only rebuild instead of being - # dropped with the rest of the committed graph. - incremental_mode = existing_graph_path.exists() if has_path else False - # --force: full scan, not the manifest-gated incremental diff — a warm - # unchanged tree would otherwise dispatch zero files (#1894). - incremental_mode = incremental_mode and not force - if force: - print("[graphify extract] --force: full re-scan, semantic cache reads skipped") - elif incremental_mode and not manifest_path.exists(): - print( - "[graphify extract] manifest.json missing; using existing " - "graph.json as the incremental baseline (all files re-checked; " - "nodes for files outside this run's scope are preserved)" + from graphify.detect import ( + detect as _detect, + detect_incremental as _detect_incremental, + save_manifest as _save_manifest, ) + manifest_path = graphify_out / "manifest.json" + existing_graph_path = graphify_out / "graph.json" + # #1925: a missing manifest.json must not degrade to a full scan that + # discards the existing graph's semantic layer. An existing graph.json + # is a sufficient incremental baseline: detect_incremental treats an + # absent manifest as "everything is new" (re-extract all, nothing + # deleted), and build_merge + _stale_graph_sources reconcile replaced + # and genuinely-deleted sources against the current corpus, so doc/ + # paper/image nodes survive a --code-only rebuild instead of being + # dropped with the rest of the committed graph. + incremental_mode = existing_graph_path.exists() if has_path else False + # --force: full scan, not the manifest-gated incremental diff — a warm + # unchanged tree would otherwise dispatch zero files (#1894). + incremental_mode = incremental_mode and not force + if force: + print("[graphify extract] --force: full re-scan, semantic cache reads skipped") + elif incremental_mode and not manifest_path.exists(): + print( + "[graphify extract] manifest.json missing; using existing " + "graph.json as the incremental baseline (all files re-checked; " + "nodes for files outside this run's scope are preserved)" + ) - if not has_path: - detection = {} - code_files = [] - doc_files = [] - paper_files = [] - image_files = [] - deleted_files = [] - excluded_files = [] - graph_stale_sources = [] - unchanged_total = 0 - files_by_type = {} - elif incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental( - target, - manifest_path=str(manifest_path), - google_workspace=google_workspace or None, - extra_excludes=_effective_excludes or None, - gitignore=_effective_gitignore, - ) - files_by_type = detection.get("files", {}) - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - excluded_files = list(detection.get("excluded_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - # #1909: derive the prune set from the existing graph itself, not - # just the manifest. A file that became excluded without ever - # being manifest-listed (every pre-#1897 graph is in this state) - # still has stale nodes carried forward by build_merge unless the - # graph's own sources are reconciled against the current corpus. - _seen_files = {f for _fl in files_by_type.values() for f in _fl} - _seen_files.update(detection.get("unclassified", [])) - graph_stale_sources = _stale_graph_sources( - existing_graph_path, target, _seen_files, detection=detection - ) - # #2543 heal: manifests poisoned BEFORE failed-source unstamping - # existed carry live hashes for code files whose extraction failed - # (missing extra, crash) — stamped up-to-date yet absent from - # graph.json, so the incremental gate skips them forever. Treat - # such a file as changed and re-queue it; if it fails again this - # run it is now left unstamped, so this cannot wedge. - _healed_sources = _zero_node_stamped_code_sources( - existing_graph_path, - target, - detection.get("unchanged_files", {}).get("code", []), - ) - if _healed_sources: + if not has_path: + detection = {} + code_files = [] + doc_files = [] + paper_files = [] + image_files = [] + deleted_files = [] + excluded_files = [] + graph_stale_sources = [] + unchanged_total = 0 + files_by_type = {} + elif incremental_mode: + print(f"[graphify extract] incremental scan of {target}") + detection = _detect_incremental( + target, + manifest_path=str(manifest_path), + google_workspace=google_workspace or None, + extra_excludes=_effective_excludes or None, + gitignore=_effective_gitignore, + ) + files_by_type = detection.get("files", {}) + new_by_type = detection.get("new_files", {}) + code_files = [Path(p) for p in new_by_type.get("code", [])] + doc_files = [Path(p) for p in new_by_type.get("document", [])] + paper_files = [Path(p) for p in new_by_type.get("paper", [])] + image_files = [Path(p) for p in new_by_type.get("image", [])] + deleted_files = list(detection.get("deleted_files", [])) + excluded_files = list(detection.get("excluded_files", [])) + unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) + # #1909: derive the prune set from the existing graph itself, not + # just the manifest. A file that became excluded without ever + # being manifest-listed (every pre-#1897 graph is in this state) + # still has stale nodes carried forward by build_merge unless the + # graph's own sources are reconciled against the current corpus. + _seen_files = {f for _fl in files_by_type.values() for f in _fl} + _seen_files.update(detection.get("unclassified", [])) + graph_stale_sources = _stale_graph_sources( + existing_graph_path, target, _seen_files, detection=detection + ) + # #2543 heal: manifests poisoned BEFORE failed-source unstamping + # existed carry live hashes for code files whose extraction failed + # (missing extra, crash) — stamped up-to-date yet absent from + # graph.json, so the incremental gate skips them forever. Treat + # such a file as changed and re-queue it; if it fails again this + # run it is now left unstamped, so this cannot wedge. + _healed_sources = _zero_node_stamped_code_sources( + existing_graph_path, + target, + detection.get("unchanged_files", {}).get("code", []), + ) + if _healed_sources: + print( + f"[graphify extract] re-queuing {len(_healed_sources)} " + f"manifest-stamped code file(s) with no nodes in graph.json " + f"(prior failed extraction, #2543)" + ) + code_files.extend(Path(p) for p in _healed_sources) + else: + print(f"[graphify extract] scanning {target}") + detection = _detect( + target, + google_workspace=google_workspace or None, + extra_excludes=_effective_excludes or None, + cache_root=out_root, + gitignore=_effective_gitignore, + ) + files_by_type = detection.get("files", {}) + code_files = [Path(p) for p in files_by_type.get("code", [])] + doc_files = [Path(p) for p in files_by_type.get("document", [])] + paper_files = [Path(p) for p in files_by_type.get("paper", [])] + image_files = [Path(p) for p in files_by_type.get("image", [])] + deleted_files = [] + excluded_files = [] + graph_stale_sources = [] + unchanged_total = 0 + + semantic_files = doc_files + paper_files + image_files + # --code-only: index code (pure local AST, no key) and skip the semantic + # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no + # LLM backend is configured (#1734). Report what was skipped rather than + # silently dropping it. + if code_only and semantic_files: print( - f"[graphify extract] re-queuing {len(_healed_sources)} " - f"manifest-stamped code file(s) with no nodes in graph.json " - f"(prior failed extraction, #2543)" + f"[graphify extract] --code-only: skipping {len(semantic_files)} " + f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images) — no LLM extraction" ) - code_files.extend(Path(p) for p in _healed_sources) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect( - target, - google_workspace=google_workspace or None, - extra_excludes=_effective_excludes or None, - cache_root=out_root, - gitignore=_effective_gitignore, - ) - files_by_type = detection.get("files", {}) - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - excluded_files = [] - graph_stale_sources = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - # --code-only: index code (pure local AST, no key) and skip the semantic - # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no - # LLM backend is configured (#1734). Report what was skipped rather than - # silently dropping it. - if code_only and semantic_files: - print( - f"[graphify extract] --code-only: skipping {len(semantic_files)} " - f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images) — no LLM extraction" - ) - semantic_files = [] - doc_files = [] - paper_files = [] - image_files = [] - if deep_mode and incremental_mode and not code_only: - # Deep mode reads/writes its own cache namespace - # (cache/semantic-deep/), so the manifest's changed-file gate is - # not a valid proxy for deep coverage: over a warm unchanged tree - # it dispatches zero files and `--mode deep` silently no-ops - # (#1894). Widen the semantic pass to the FULL live - # doc/paper/image set (``files_by_type`` from detect_incremental, - # which already excludes excluded files) and let the - # mode-namespaced cache decide hits/misses — the first deep run - # re-dispatches everything (deep namespace cold), later deep runs - # hit the deep cache. - _deep_all = [ - Path(p) - for _ftype in ("document", "paper", "image") - for p in files_by_type.get(_ftype, []) - ] - if len(_deep_all) != len(semantic_files): + semantic_files = [] + doc_files = [] + paper_files = [] + image_files = [] + if deep_mode and incremental_mode and not code_only: + # Deep mode reads/writes its own cache namespace + # (cache/semantic-deep/), so the manifest's changed-file gate is + # not a valid proxy for deep coverage: over a warm unchanged tree + # it dispatches zero files and `--mode deep` silently no-ops + # (#1894). Widen the semantic pass to the FULL live + # doc/paper/image set (``files_by_type`` from detect_incremental, + # which already excludes excluded files) and let the + # mode-namespaced cache decide hits/misses — the first deep run + # re-dispatches everything (deep namespace cold), later deep runs + # hit the deep cache. + _deep_all = [ + Path(p) + for _ftype in ("document", "paper", "image") + for p in files_by_type.get(_ftype, []) + ] + if len(_deep_all) != len(semantic_files): + print( + f"[graphify extract] deep mode: widening semantic pass from " + f"{len(semantic_files)} changed to {len(_deep_all)} live " + f"doc/paper/image file(s); the deep semantic cache decides " + f"what is re-extracted" + ) + semantic_files = _deep_all + if incremental_mode: + # Excluded-but-alive files are reported separately from deletions + # (#1908): they still exist on disk, the scan just stopped + # covering them (ignore rules / --exclude changed). + _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" print( - f"[graphify extract] deep mode: widening semantic pass from " - f"{len(semantic_files)} changed to {len(_deep_all)} live " - f"doc/paper/image file(s); the deep semantic cache decides " - f"what is re-extracted" + f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " + f"{len(paper_files)} papers, {len(image_files)} images changed; " + f"{unchanged_total} unchanged; {len(deleted_files)} deleted" + f"{_excl_note}" ) - semantic_files = _deep_all - if incremental_mode: - # Excluded-but-alive files are reported separately from deletions - # (#1908): they still exist on disk, the scan just stopped - # covering them (ignore rules / --exclude changed). - _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - f"{_excl_note}" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) - # Surface files that were seen but not classified (extensionless non-shebang - # project files like Dockerfile/Makefile, or unsupported extensions), so they - # are no longer invisible in graphify's own output (#1692). - _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] - if _unclassified: - _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) - _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" - print( - f"[graphify extract] {len(_unclassified)} file(s) not classified " - f"(no supported extension or shebang), skipped: {_names}{_more}" - ) - # Name the files dropped by the sensitive-file filter so a wrongly-flagged - # source/doc is visible, not just a count (#2106). Operational skips - # (symlink/office/Workspace) carry a " [reason]" suffix; exclude those here - # so this line reports only the security-heuristic drops. - _sensitive = detection.get("skipped_sensitive", []) if isinstance(detection, dict) else [] - _sec = [s for s in _sensitive if " [" not in s] - if _sec: - _snames = ", ".join(sorted({Path(p).name for p in _sec})[:6]) - _smore = f" (+{len(_sec) - 6} more)" if len(_sec) > 6 else "" - print( - f"[graphify extract] {len(_sec)} file(s) skipped as potentially sensitive " - f"(rename or move if wrongly flagged): {_snames}{_smore}" - ) - stages.mark("detect") - - # Resolve the LLM backend only now that we know whether the corpus - # needs one. A code-only corpus is pure local AST and must not require - # an API key; the key is enforced below only when there's LLM work. - from graphify.llm import ( - BACKENDS as _BACKENDS, - detect_backend as _detect_backend, - estimate_cost as _estimate_cost, - extract_corpus_parallel as _extract_corpus_parallel, - _format_backend_env_keys, - _get_backend_api_key, - ) - needs_llm = bool(semantic_files) or dedup_llm - if backend is None and needs_llm: - backend = _detect_backend() - if backend is not None and backend not in _BACKENDS: - print( - f"error: unknown backend '{backend}'. " - f"Available: {', '.join(sorted(_BACKENDS))}", - file=sys.stderr, + else: + print( + f"[graphify extract] found {len(code_files)} code, " + f"{len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images" + ) + # Surface files that were seen but not classified (extensionless non-shebang + # project files like Dockerfile/Makefile, or unsupported extensions), so they + # are no longer invisible in graphify's own output (#1692). + _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] + if _unclassified: + _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) + _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" + print( + f"[graphify extract] {len(_unclassified)} file(s) not classified " + f"(no supported extension or shebang), skipped: {_names}{_more}" + ) + # Name the files dropped by the sensitive-file filter so a wrongly-flagged + # source/doc is visible, not just a count (#2106). Operational skips + # (symlink/office/Workspace) carry a " [reason]" suffix; exclude those here + # so this line reports only the security-heuristic drops. + _sensitive = detection.get("skipped_sensitive", []) if isinstance(detection, dict) else [] + _sec = [s for s in _sensitive if " [" not in s] + if _sec: + _snames = ", ".join(sorted({Path(p).name for p in _sec})[:6]) + _smore = f" (+{len(_sec) - 6} more)" if len(_sec) > 6 else "" + print( + f"[graphify extract] {len(_sec)} file(s) skipped as potentially sensitive " + f"(rename or move if wrongly flagged): {_snames}{_smore}" + ) + stages.mark("detect") + + # Resolve the LLM backend only now that we know whether the corpus + # needs one. A code-only corpus is pure local AST and must not require + # an API key; the key is enforced below only when there's LLM work. + from graphify.llm import ( + BACKENDS as _BACKENDS, + detect_backend as _detect_backend, + estimate_cost as _estimate_cost, + extract_corpus_parallel as _extract_corpus_parallel, + _format_backend_env_keys, + _get_backend_api_key, ) - sys.exit(1) - if needs_llm: - if backend is None: - reasons = [] - if semantic_files: - reasons.append( - f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" - ) - if dedup_llm: - reasons.append("--dedup-llm was passed") - hint = "" - if semantic_files: - hint = (" Or pass --code-only to index just the code " - "(local AST, no key) and skip the non-code files.") + needs_llm = bool(semantic_files) or dedup_llm + if backend is None and needs_llm: + backend = _detect_backend() + if backend is not None and backend not in _BACKENDS: print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key." + hint, + f"error: unknown backend '{backend}'. " + f"Available: {', '.join(sorted(_BACKENDS))}", file=sys.stderr, ) sys.exit(1) - if backend == "ollama": - from graphify.llm import _validate_ollama_base_url - _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) - try: - _validate_ollama_base_url(_oll_url, warn=False) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(2) - if not _get_backend_api_key(backend): - allow_no_key = False - if backend == "ollama": - from urllib.parse import urlparse - ollama_url = os.environ.get( - "OLLAMA_BASE_URL", - _BACKENDS["ollama"].get("base_url", ""), + # Validate the fallback's NAME upfront so a typo fails before any API + # spend; its key/credential check is deferred to fire time — if the + # retry then fails too, the total-failure error below already tells + # the user what to install or set. + if fallback_backend is not None and fallback_backend not in _BACKENDS: + print( + f"error: unknown fallback backend '{fallback_backend}'. " + f"Available: {', '.join(sorted(_BACKENDS))}", + file=sys.stderr, + ) + sys.exit(1) + if needs_llm: + if backend is None: + reasons = [] + if semantic_files: + reasons.append( + f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" + ) + if dedup_llm: + reasons.append("--dedup-llm was passed") + hint = "" + if semantic_files: + hint = (" Or pass --code-only to index just the code " + "(local AST, no key) and skip the non-code files.") + print( + "error: no LLM API key found (" + "; ".join(reasons) + "). " + "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " + "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " + "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " + "corpus needs no key." + hint, + file=sys.stderr, ) + sys.exit(1) + if backend == "ollama": + from graphify.llm import _validate_ollama_base_url + _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) try: - host = (urlparse(ollama_url).hostname or "").lower() - except Exception: - host = "" - allow_no_key = ( - host in ("localhost", "127.0.0.1", "::1") - or host.startswith("127.") - ) - elif backend == "bedrock": - allow_no_key = bool( - os.environ.get("AWS_PROFILE") - or os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION") - or os.environ.get("AWS_ACCESS_KEY_ID") - ) - elif backend == "claude-cli": - import shutil as _shutil - allow_no_key = _shutil.which("claude") is not None + _validate_ollama_base_url(_oll_url, warn=False) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(2) + if not _get_backend_api_key(backend): + allow_no_key = False + if backend == "ollama": + from urllib.parse import urlparse + ollama_url = os.environ.get( + "OLLAMA_BASE_URL", + _BACKENDS["ollama"].get("base_url", ""), + ) + try: + host = (urlparse(ollama_url).hostname or "").lower() + except Exception: + host = "" + allow_no_key = ( + host in ("localhost", "127.0.0.1", "::1") + or host.startswith("127.") + ) + elif backend == "bedrock": + allow_no_key = bool( + os.environ.get("AWS_PROFILE") + or os.environ.get("AWS_REGION") + or os.environ.get("AWS_DEFAULT_REGION") + or os.environ.get("AWS_ACCESS_KEY_ID") + ) + elif backend == "claude-cli": + import shutil as _shutil + allow_no_key = _shutil.which("claude") is not None + if not allow_no_key: + print( + "error: backend 'claude-cli' requires the `claude` CLI on $PATH " + "(install Claude Code and run `claude` once to authenticate).", + file=sys.stderr, + ) + sys.exit(1) + elif backend == "openai-cli": + import shutil as _shutil + allow_no_key = _shutil.which("codex") is not None + if not allow_no_key: + print( + "error: backend 'openai-cli' requires the `codex` CLI on $PATH " + "(npm install -g @openai/codex, then run `codex` once to " + "authenticate with your ChatGPT account).", + file=sys.stderr, + ) + sys.exit(1) if not allow_no_key: print( - "error: backend 'claude-cli' requires the `claude` CLI on $PATH " - "(install Claude Code and run `claude` once to authenticate).", + f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", file=sys.stderr, ) sys.exit(1) - if not allow_no_key: - print( - f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", - file=sys.stderr, - ) - sys.exit(1) - # Track whether this run's extraction was incomplete (a whole extractor - # pass crashed, or some semantic chunks failed). A partial result must not - # be force-written over a good complete graph — the final write falls back - # to the #479 shrink guard unless --allow-partial is set. - _extraction_incomplete = False - # A walk that couldn't fully enumerate the corpus (permission-denied - # subtree, I/O error) yields a legitimately smaller graph that must not - # be force-written over a complete one — same failure class as a crashed - # pass. detect()/detect_incremental() already record these; consume them. - if detection.get("walk_errors"): - _extraction_incomplete = True - - # AST extraction on code files. Empty code list (docs-only corpus) is - # the issue #698 case — skip cleanly instead of crashing inside extract(). - ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - if code_files: - from graphify.extract import extract as _ast_extract - # Anchor the cache at the output root, not the scanned project: - # with --out, a /graphify-out/cache/ would leak a - # graphify-out/ dir into a project that asked for external output. - # `root` stays the scanned project so source_file/ids relativize - # against it; conflating the two basenamed every node (#1941). - ast_kwargs: dict = {"cache_root": out_root, "root": target} - if cli_max_workers is not None: - ast_kwargs["max_workers"] = cli_max_workers - # #2437/#2438 (the `graphify update` twin of watch's #2406 fix): an - # incremental re-scan extracts only the changed code files, so the - # cross-file resolvers cannot see a callee living in an unchanged - # file and every changed->unchanged call edge silently vanished on - # merge. Hand extract() read-only resolution context from the - # persisted graph: its AST-tier nodes (with their `_callable`/ - # `_callable_class` markers, #2438) plus the contains/method edges - # the member-call resolvers walk (#2437), scoped to the UNCHANGED - # live corpus — never a re-extracted, deleted, or excluded file, so - # stale symbols cannot resurrect. Fails open (changed-batch-only - # resolution, the pre-fix behavior) on an unreadable graph. - if incremental_mode and existing_graph_path.exists(): - _ctx_nodes: list[dict] = [] - _ctx_edges: list[dict] = [] + # Track whether this run's extraction was incomplete (a whole extractor + # pass crashed, or some semantic chunks failed). A partial result must not + # be force-written over a good complete graph — the final write falls back + # to the #479 shrink guard unless --allow-partial is set. + _extraction_incomplete = False + # A walk that couldn't fully enumerate the corpus (permission-denied + # subtree, I/O error) yields a legitimately smaller graph that must not + # be force-written over a complete one — same failure class as a crashed + # pass. detect()/detect_incremental() already record these; consume them. + if detection.get("walk_errors"): + _extraction_incomplete = True + + # AST extraction on code files. Empty code list (docs-only corpus) is + # the issue #698 case — skip cleanly instead of crashing inside extract(). + ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + if code_files: + from graphify.extract import extract as _ast_extract + # Anchor the cache at the output root, not the scanned project: + # with --out, a /graphify-out/cache/ would leak a + # graphify-out/ dir into a project that asked for external output. + # `root` stays the scanned project so source_file/ids relativize + # against it; conflating the two basenamed every node (#1941). + ast_kwargs: dict = {"cache_root": out_root, "root": target} + if cli_max_workers is not None: + ast_kwargs["max_workers"] = cli_max_workers + # #2437/#2438 (the `graphify update` twin of watch's #2406 fix): an + # incremental re-scan extracts only the changed code files, so the + # cross-file resolvers cannot see a callee living in an unchanged + # file and every changed->unchanged call edge silently vanished on + # merge. Hand extract() read-only resolution context from the + # persisted graph: its AST-tier nodes (with their `_callable`/ + # `_callable_class` markers, #2438) plus the contains/method edges + # the member-call resolvers walk (#2437), scoped to the UNCHANGED + # live corpus — never a re-extracted, deleted, or excluded file, so + # stale symbols cannot resurrect. Fails open (changed-batch-only + # resolution, the pre-fix behavior) on an unreadable graph. + if incremental_mode and existing_graph_path.exists(): + _ctx_nodes: list[dict] = [] + _ctx_edges: list[dict] = [] + try: + from graphify.build import _is_ast_tier as _ctx_is_ast_tier + from graphify.security import ( + check_graph_file_size_cap as _ctx_size_cap, + ) + _ctx_size_cap(existing_graph_path) + _ctx_graph = json.loads( + existing_graph_path.read_text(encoding="utf-8") + ) + _ctx_root = Path(os.path.abspath(target)) + + def _ctx_identity(source_file) -> str | None: + # graph.json source_file values are relative to the + # scanned root (`root=target` above); detect's + # unchanged_files keep their scan-time form. Compare + # both as absolute posix paths. + if not source_file: + return None + _p = Path(str(source_file)) + if not _p.is_absolute(): + _p = _ctx_root / _p + return Path(os.path.abspath(_p)).as_posix() + + _ctx_live = { + _ctx_identity(f) + for _flist in detection.get("unchanged_files", {}).values() + for f in _flist + } + _ctx_live.discard(None) + for _node in _ctx_graph.get("nodes", []): + if not _node.get("id") or not _ctx_is_ast_tier(_node): + continue + _sf = _node.get("source_file") + if not _sf or _ctx_identity(_sf) not in _ctx_live: + continue + _ctx_node = { + "id": _node["id"], + "label": _node.get("label"), + "source_file": _sf, + "file_type": _node.get("file_type"), + "type": _node.get("type"), + } + for _marker in ("_callable", "_callable_class"): + if _node.get(_marker): + _ctx_node[_marker] = _node[_marker] + _ctx_nodes.append(_ctx_node) + for _edge in _ctx_graph.get( + "links", _ctx_graph.get("edges", []) + ): + if _edge.get("relation") not in ("contains", "method"): + continue + if not _ctx_is_ast_tier(_edge): + continue + _sf = _edge.get("source_file") + if not _sf or _ctx_identity(_sf) not in _ctx_live: + continue + _ctx_edges.append({ + "source": _edge.get("source"), + "target": _edge.get("target"), + "relation": _edge.get("relation"), + "source_file": _sf, + }) + except Exception: + _ctx_nodes, _ctx_edges = [], [] + if _ctx_nodes: + ast_kwargs["resolution_context_nodes"] = _ctx_nodes + if _ctx_edges: + ast_kwargs["resolution_context_edges"] = _ctx_edges + print(f"[graphify extract] AST extraction on {len(code_files)} code files...") try: - from graphify.build import _is_ast_tier as _ctx_is_ast_tier - from graphify.security import ( - check_graph_file_size_cap as _ctx_size_cap, - ) - _ctx_size_cap(existing_graph_path) - _ctx_graph = json.loads( - existing_graph_path.read_text(encoding="utf-8") + ast_result = _ast_extract(code_files, **ast_kwargs) + except Exception as exc: + print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) + # #2445: losing the whole AST pass is fatal by default. The + # empty stand-in only reaches the shrink guard when an existing + # graph is larger — on a fresh build it used to be written as a + # 0-node graph with exit 0, indistinguishable from success. + # --allow-partial opts back into the best-effort continuation. + if not cli_allow_partial: + sys.exit(1) + ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + _extraction_incomplete = True # the whole AST pass was lost + stages.mark("AST extract") + + # Semantic extraction on docs/papers/images. Check cache first. + from graphify.cache import ( + check_semantic_cache as _check_semantic_cache, + prune_semantic_cache as _prune_semantic_cache, + save_semantic_cache as _save_semantic_cache, + ) + sem_result: dict = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + } + # Semantic files whose extraction truncated this run. They are left + # unstamped in the manifest so detect_incremental re-queues them next run + # (mirrors the #933 failed-chunk handling); captured below before the + # _partial markers are stripped from the corpus. + _partial_semantic_files: set[str] = set() + sem_cache_hits = 0 + sem_cache_misses = 0 + # Deep mode uses its own namespace (cache/semantic-deep/) so deep and + # standard results for the same content never shadow each other (#1894). + sem_cache_mode = "deep" if deep_mode else None + # Entries are attributed to the extraction prompt that produced them, so + # a release that changes the prompt re-extracts rather than replaying the + # older vintage alongside the new one (#1939). Read and write must pass + # the same prompt, or the write lands where the next read won't look. + from graphify.llm import _extraction_system as _sem_prompt_for + sem_prompt = _sem_prompt_for(deep=deep_mode) + if semantic_files: + sem_paths_str = [str(p) for p in semantic_files] + if force: + # --force: skip the cache READ so every semantic file is + # re-dispatched; the save below still runs so the fresh + # results replace the stale entries. + cached_nodes, cached_edges, cached_hyperedges = [], [], [] + uncached_paths = list(sem_paths_str) + else: + cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( + _check_semantic_cache(sem_paths_str, root=target, cache_root=out_root, + mode=sem_cache_mode, prompt=sem_prompt) ) - _ctx_root = Path(os.path.abspath(target)) - - def _ctx_identity(source_file) -> str | None: - # graph.json source_file values are relative to the - # scanned root (`root=target` above); detect's - # unchanged_files keep their scan-time form. Compare - # both as absolute posix paths. - if not source_file: - return None - _p = Path(str(source_file)) - if not _p.is_absolute(): - _p = _ctx_root / _p - return Path(os.path.abspath(_p)).as_posix() - - _ctx_live = { - _ctx_identity(f) - for _flist in detection.get("unchanged_files", {}).values() - for f in _flist - } - _ctx_live.discard(None) - for _node in _ctx_graph.get("nodes", []): - if not _node.get("id") or not _ctx_is_ast_tier(_node): - continue - _sf = _node.get("source_file") - if not _sf or _ctx_identity(_sf) not in _ctx_live: - continue - _ctx_node = { - "id": _node["id"], - "label": _node.get("label"), - "source_file": _sf, - "file_type": _node.get("file_type"), - "type": _node.get("type"), + sem_cache_hits = len(semantic_files) - len(uncached_paths) + sem_cache_misses = len(uncached_paths) + sem_result["nodes"].extend(cached_nodes) + sem_result["edges"].extend(cached_edges) + sem_result["hyperedges"].extend(cached_hyperedges) + if sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") + + if uncached_paths: + def _dispatch_semantic( + be: str, paths: list[str], *, last_resort: bool = True + ) -> tuple[dict, dict]: + """Run one semantic-extraction pass over ``paths`` via ``be``. + + Returns ``(fresh, chunk_stats)``. ``chunk_stats`` counts + per-chunk successes via the progress callback (issue #792 + addendum: it also keeps the CLI from being silent during + long local-inference runs) and records ``crashed`` when the + whole pass raised. A crashed pass returns an empty + accumulator instead of propagating, so the caller can retry + the same paths on --fallback-backend before failing the + build. ``last_resort=False`` softens a missing-SDK + ImportError from fatal to a failed pass — with a fallback + configured, a missing package on the primary is exactly the + case the fallback exists for. + """ + print(f"[graphify extract] semantic extraction on {len(paths)} files via {be}...") + corpus_kwargs: dict = { + "backend": be, + # --model names a model on the PRIMARY backend; on the + # fallback it would be an unknown name there, so the + # fallback runs on its own default model. + "model": model if be == backend else None, + "root": target, + "cache_root": out_root, } - for _marker in ("_callable", "_callable_class"): - if _node.get(_marker): - _ctx_node[_marker] = _node[_marker] - _ctx_nodes.append(_ctx_node) - for _edge in _ctx_graph.get( - "links", _ctx_graph.get("edges", []) - ): - if _edge.get("relation") not in ("contains", "method"): - continue - if not _ctx_is_ast_tier(_edge): - continue - _sf = _edge.get("source_file") - if not _sf or _ctx_identity(_sf) not in _ctx_live: - continue - _ctx_edges.append({ - "source": _edge.get("source"), - "target": _edge.get("target"), - "relation": _edge.get("relation"), - "source_file": _sf, - }) - except Exception: - _ctx_nodes, _ctx_edges = [], [] - if _ctx_nodes: - ast_kwargs["resolution_context_nodes"] = _ctx_nodes - if _ctx_edges: - ast_kwargs["resolution_context_edges"] = _ctx_edges - print(f"[graphify extract] AST extraction on {len(code_files)} code files...") + if deep_mode: + corpus_kwargs["deep_mode"] = True + if cli_token_budget is not None: + corpus_kwargs["token_budget"] = cli_token_budget + if cli_max_concurrency is not None: + corpus_kwargs["max_concurrency"] = cli_max_concurrency + + chunk_stats = {"total": 0, "succeeded": 0, "crashed": False} + def _progress(idx: int, total: int, _result: dict) -> None: + chunk_stats["total"] = total + chunk_stats["succeeded"] += 1 + print( + f"[graphify extract] chunk {idx + 1}/{total} done", + flush=True, + ) + corpus_kwargs["on_chunk_done"] = _progress + + _empty = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + try: + fresh = _extract_corpus_parallel( + [Path(p) for p in paths], + **corpus_kwargs, + ) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + if last_resort: + sys.exit(1) + fresh = dict(_empty) + chunk_stats["crashed"] = True + except Exception as exc: + print( + f"[graphify extract] semantic extraction failed: {exc}", + file=sys.stderr, + ) + fresh = dict(_empty) + chunk_stats["crashed"] = True # the semantic pass crashed + return fresh, chunk_stats + + _fallback_eligible = ( + fallback_backend is not None and fallback_backend != backend + ) + fresh, _chunk_stats = _dispatch_semantic( + backend, uncached_paths, last_resort=not _fallback_eligible + ) + _last_backend = backend + if _fallback_eligible and _chunk_stats["succeeded"] == 0: + # Nothing was cache-saved for a zero-success pass (the save + # runs below), so the fallback retries exactly the same + # still-uncached files, once. + print( + f"[graphify extract] all semantic chunks failed for backend " + f"'{backend}'; retrying once with fallback backend " + f"'{fallback_backend}'..." + ) + fresh, _chunk_stats = _dispatch_semantic( + fallback_backend, uncached_paths + ) + _last_backend = fallback_backend + + # on_chunk_done only fires after a chunk succeeds. If fresh + # semantic extraction was requested and no chunks completed + # (on the fallback either, when one was configured), fail + # instead of writing an AST-only graph with exit 0. + if uncached_paths and _chunk_stats["succeeded"] == 0: + print( + f"[graphify extract] error: all semantic chunks failed " + f"for backend '{_last_backend}' ({len(uncached_paths)} uncached files) - " + f"see per-chunk errors above. If you see 'requires the X package', " + f"run `pip install X` and retry.", + file=sys.stderr, + ) + sys.exit(1) + # Incompleteness is judged on the pass whose result we kept: a + # crashed pass, or some (but not all) chunks failed — the graph + # is missing nodes from the failed chunks, so it must not + # clobber a larger complete graph without an explicit + # --allow-partial override. + if _chunk_stats["crashed"]: + _extraction_incomplete = True + if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]: + _extraction_incomplete = True + # Which files truncated this run (item markers + the empty-parse + # _partial_files set). Computed BEFORE the save so it can be passed + # as partial_source_files: without it, a file whose only truncated + # chunk parsed empty (so it has no item markers here) would be + # written as a complete cache entry, re-promoting it (#1950). + from graphify.llm import ( + _partial_source_files as _partial_sf, + _strip_partial_markers as _strip_partial, + ) + _partial_semantic_files = set(_partial_sf(fresh)) + try: + _save_semantic_cache( + fresh.get("nodes", []), + fresh.get("edges", []), + fresh.get("hyperedges", []), + root=target, + cache_root=out_root, + allowed_source_files=uncached_paths, + mode=sem_cache_mode, + prompt=sem_prompt, + partial_source_files=_partial_semantic_files or None, + ) + except Exception as exc: + print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) + # Strip the markers before the corpus feeds the graph so the + # internal flag never leaks into graph.json. + _strip_partial(fresh) + sem_result["nodes"].extend(fresh.get("nodes", [])) + sem_result["edges"].extend(fresh.get("edges", [])) + sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) + sem_result["input_tokens"] += fresh.get("input_tokens", 0) + sem_result["output_tokens"] += fresh.get("output_tokens", 0) + + # Prune orphaned semantic cache entries. The semantic cache is + # content-hash-keyed and unversioned, so it is never swept by the AST + # version-cleanup: every content change or file deletion leaves a + # permanent orphan that accumulates unbounded (#1527). Sweep it against + # the FULL live document set (``files_by_type`` — present in both the + # incremental and full branches), NOT the incremental ``semantic_files`` + # changed-subset, which would delete every unchanged doc's valid entry. + # Best-effort: a prune failure must never break extraction. + # Hash keys are anchored to the corpus (``target``) — the same anchor + # the cache read/write above use — while the stat-index artifact + # follows the cache location (``out_root``). Anchoring these hashes to + # ``out_root`` instead would mismatch every key under ``--out`` and + # sweep the entire fresh cache as orphaned (#1990/#1991). try: - ast_result = _ast_extract(code_files, **ast_kwargs) + from graphify.cache import file_hash as _file_hash + _live_hashes: set[str] = set() + for _kind in ("document", "paper", "image"): + for _fp in files_by_type.get(_kind, []): + _abs = Path(_fp) + if not _abs.is_absolute(): + _abs = Path(target) / _abs + if not _abs.is_file(): + continue # deleted/missing — leave out so its entry is pruned + try: + _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) + except OSError: + pass + # A pathless database extraction has no filesystem corpus to sweep. + if has_path: + _prune_semantic_cache(out_root, _live_hashes) except Exception as exc: - print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) - # #2445: losing the whole AST pass is fatal by default. The - # empty stand-in only reaches the shrink guard when an existing - # graph is larger — on a fresh build it used to be written as a - # 0-node graph with exit 0, indistinguishable from success. - # --allow-partial opts back into the best-effort continuation. - if not cli_allow_partial: - sys.exit(1) - ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - _extraction_incomplete = True # the whole AST pass was lost - stages.mark("AST extract") - - # Semantic extraction on docs/papers/images. Check cache first. - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - prune_semantic_cache as _prune_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - # Semantic files whose extraction truncated this run. They are left - # unstamped in the manifest so detect_incremental re-queues them next run - # (mirrors the #933 failed-chunk handling); captured below before the - # _partial markers are stripped from the corpus. - _partial_semantic_files: set[str] = set() - sem_cache_hits = 0 - sem_cache_misses = 0 - # Deep mode uses its own namespace (cache/semantic-deep/) so deep and - # standard results for the same content never shadow each other (#1894). - sem_cache_mode = "deep" if deep_mode else None - # Entries are attributed to the extraction prompt that produced them, so - # a release that changes the prompt re-extracts rather than replaying the - # older vintage alongside the new one (#1939). Read and write must pass - # the same prompt, or the write lands where the next read won't look. - from graphify.llm import _extraction_system as _sem_prompt_for - sem_prompt = _sem_prompt_for(deep=deep_mode) - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - if force: - # --force: skip the cache READ so every semantic file is - # re-dispatched; the save below still runs so the fresh - # results replace the stale entries. - cached_nodes, cached_edges, cached_hyperedges = [], [], [] - uncached_paths = list(sem_paths_str) - else: - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=target, cache_root=out_root, - mode=sem_cache_mode, prompt=sem_prompt) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - corpus_kwargs: dict = { - "backend": backend, - "model": model, - "root": target, - "cache_root": out_root, - } - if deep_mode: - corpus_kwargs["deep_mode"] = True - if cli_token_budget is not None: - corpus_kwargs["token_budget"] = cli_token_budget - if cli_max_concurrency is not None: - corpus_kwargs["max_concurrency"] = cli_max_concurrency - - # Minimal progress callback so the CLI is no longer silent - # during long local-inference runs (issue #792 addendum). - # Also track per-chunk success so we can fail loudly when - # every chunk errors (e.g. missing backend SDK package). - _chunk_stats = {"total": 0, "succeeded": 0} - def _progress(idx: int, total: int, _result: dict) -> None: - _chunk_stats["total"] = total - _chunk_stats["succeeded"] += 1 - print( - f"[graphify extract] chunk {idx + 1}/{total} done", - flush=True, - ) - corpus_kwargs["on_chunk_done"] = _progress + print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) + stages.mark("semantic extract") + pg_result: dict = {"nodes": [], "edges": []} + if cli_postgres_dsn is not None: + from graphify.pg_introspect import introspect_postgres + print(f"[graphify extract] introspecting PostgreSQL schema...") try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - **corpus_kwargs, - ) - except ImportError as exc: + pg_result = introspect_postgres(cli_postgres_dsn) + except (ConnectionError, ImportError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) - except Exception as exc: - print( - f"[graphify extract] semantic extraction failed: {exc}", - file=sys.stderr, - ) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - _extraction_incomplete = True # the semantic pass crashed + print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " + f"{len(pg_result['edges'])} edges") - # on_chunk_done only fires after a chunk succeeds. If fresh - # semantic extraction was requested and no chunks completed, - # fail instead of writing an AST-only graph with exit 0. - if uncached_paths and _chunk_stats["succeeded"] == 0: - print( - f"[graphify extract] error: all semantic chunks failed " - f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " - f"see per-chunk errors above. If you see 'requires the X package', " - f"run `pip install X` and retry.", - file=sys.stderr, - ) - sys.exit(1) - # Some (but not all) chunks failed — the graph is missing nodes - # from the failed chunks, so it must not clobber a larger complete - # graph without an explicit --allow-partial override. - if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]: - _extraction_incomplete = True - # Which files truncated this run (item markers + the empty-parse - # _partial_files set). Computed BEFORE the save so it can be passed - # as partial_source_files: without it, a file whose only truncated - # chunk parsed empty (so it has no item markers here) would be - # written as a complete cache entry, re-promoting it (#1950). - from graphify.llm import ( - _partial_source_files as _partial_sf, - _strip_partial_markers as _strip_partial, - ) - _partial_semantic_files = set(_partial_sf(fresh)) + cargo_result: dict = {"nodes": [], "edges": []} + if cli_cargo: + from graphify.cargo_introspect import introspect_cargo + print("[graphify extract] introspecting Cargo workspace...") try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=target, - cache_root=out_root, - allowed_source_files=uncached_paths, - mode=sem_cache_mode, - prompt=sem_prompt, - partial_source_files=_partial_semantic_files or None, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - # Strip the markers before the corpus feeds the graph so the - # internal flag never leaks into graph.json. - _strip_partial(fresh) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) - - # Prune orphaned semantic cache entries. The semantic cache is - # content-hash-keyed and unversioned, so it is never swept by the AST - # version-cleanup: every content change or file deletion leaves a - # permanent orphan that accumulates unbounded (#1527). Sweep it against - # the FULL live document set (``files_by_type`` — present in both the - # incremental and full branches), NOT the incremental ``semantic_files`` - # changed-subset, which would delete every unchanged doc's valid entry. - # Best-effort: a prune failure must never break extraction. - # Hash keys are anchored to the corpus (``target``) — the same anchor - # the cache read/write above use — while the stat-index artifact - # follows the cache location (``out_root``). Anchoring these hashes to - # ``out_root`` instead would mismatch every key under ``--out`` and - # sweep the entire fresh cache as orphaned (#1990/#1991). - try: - from graphify.cache import file_hash as _file_hash - _live_hashes: set[str] = set() - for _kind in ("document", "paper", "image"): - for _fp in files_by_type.get(_kind, []): - _abs = Path(_fp) - if not _abs.is_absolute(): - _abs = Path(target) / _abs - if not _abs.is_file(): - continue # deleted/missing — leave out so its entry is pruned - try: - _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) - except OSError: - pass - # A pathless database extraction has no filesystem corpus to sweep. - if has_path: - _prune_semantic_cache(out_root, _live_hashes) - except Exception as exc: - print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) - stages.mark("semantic extract") - - pg_result: dict = {"nodes": [], "edges": []} - if cli_postgres_dsn is not None: - from graphify.pg_introspect import introspect_postgres - print(f"[graphify extract] introspecting PostgreSQL schema...") - try: - pg_result = introspect_postgres(cli_postgres_dsn) - except (ConnectionError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " - f"{len(pg_result['edges'])} edges") - - cargo_result: dict = {"nodes": [], "edges": []} - if cli_cargo: - from graphify.cargo_introspect import introspect_cargo - print("[graphify extract] introspecting Cargo workspace...") - try: - cargo_result = introspect_cargo(target) - except (ConnectionError, ImportError, OSError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " - f"{len(cargo_result['edges'])} edges") - - # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST - # first means semantic node attributes win on collision (richer labels - # for symbols also referenced in docs). Hyperedges only come from the - # semantic side. - merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), - "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), - "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), - } + cargo_result = introspect_cargo(target) + except (ConnectionError, ImportError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " + f"{len(cargo_result['edges'])} edges") + + # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST + # first means semantic node attributes win on collision (richer labels + # for symbols also referenced in docs). Hyperedges only come from the + # semantic side. + merged: dict = { + "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), + "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), + "hyperedges": list(sem_result.get("hyperedges", [])), + "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), + "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), + } - graph_json_path = graphify_out / "graph.json" - analysis_path = graphify_out / ".graphify_analysis.json" - - # Build a manifest-safe files dict: only stamp semantic_hash for files - # that actually produced output (cache hit or fresh extraction). Files - # whose chunk failed have no source_file entry in sem_result — leaving - # their semantic_hash empty so detect_incremental re-queues them (#933). - # Path normalization against the scan root happens inside the helper - # (#1897) so fresh root-relative source_files match detect()'s - # absolute file lists. - # #2543: also drop AST sources that failed (missing optional extra / - # zero-node anomaly) so they are not frozen as up-to-date. - _failed_ast_sources = list(ast_result.get("failed_sources") or []) - _manifest_files = _stamped_manifest_files( - files_by_type, - sem_result, - target, - partial_source_files=_partial_semantic_files, - failed_ast_sources=_failed_ast_sources, - ) + graph_json_path = graphify_out / "graph.json" + analysis_path = graphify_out / ".graphify_analysis.json" + + # Build a manifest-safe files dict: only stamp semantic_hash for files + # that actually produced output (cache hit or fresh extraction). Files + # whose chunk failed have no source_file entry in sem_result — leaving + # their semantic_hash empty so detect_incremental re-queues them (#933). + # Path normalization against the scan root happens inside the helper + # (#1897) so fresh root-relative source_files match detect()'s + # absolute file lists. + # #2543: also drop AST sources that failed (missing optional extra / + # zero-node anomaly) so they are not frozen as up-to-date. + _failed_ast_sources = list(ast_result.get("failed_sources") or []) + _manifest_files = _stamped_manifest_files( + files_by_type, + sem_result, + target, + partial_source_files=_partial_semantic_files, + failed_ast_sources=_failed_ast_sources, + ) - # Files dispatched this run but dropped by _stamped_manifest_files - # above (failed chunk, LLM omission, or any future exclusion) still - # carry a stale semantic_hash from a prior successful run in the - # on-disk manifest; save_manifest's seed loop would otherwise copy it - # verbatim and mask the omission (#1948). Derived from semantic_files - # — what was actually SENT to the backend this run (narrowed by the - # incremental gate and --code-only, widened by deep mode) — NOT from - # files_by_type: the full live corpus includes untouched files that - # were never dispatched, and clearing those would blank the whole - # manifest on every partial incremental run, forcing a full-corpus - # re-extraction on the next one. - _stamped_semantic = { - f for _flist in _manifest_files.values() for f in _flist - } - _cleared_semantic = {str(p) for p in semantic_files} - _stamped_semantic - # #2543: AST failures need both hashes blanked (clear_ast), not just - # semantic_hash — otherwise a prior bad stamp keeps the file "unchanged". - _cleared_ast = set(_failed_ast_sources) - - # Full-scan manifest saves prune rows for in-root files that left the - # scan corpus but still exist on disk (#1908). The corpus must be the - # RAW detect output (files_by_type), NOT the #933-stamp-filtered - # _manifest_files above — pruning to the filtered set would erase - # failed-chunk/omitted-doc rows and every doc row on --code-only runs. - _scan_corpus = ( - {f for _fl in files_by_type.values() for f in _fl} - if has_path else None - ) + # Files dispatched this run but dropped by _stamped_manifest_files + # above (failed chunk, LLM omission, or any future exclusion) still + # carry a stale semantic_hash from a prior successful run in the + # on-disk manifest; save_manifest's seed loop would otherwise copy it + # verbatim and mask the omission (#1948). Derived from semantic_files + # — what was actually SENT to the backend this run (narrowed by the + # incremental gate and --code-only, widened by deep mode) — NOT from + # files_by_type: the full live corpus includes untouched files that + # were never dispatched, and clearing those would blank the whole + # manifest on every partial incremental run, forcing a full-corpus + # re-extraction on the next one. + _stamped_semantic = { + f for _flist in _manifest_files.values() for f in _flist + } + _cleared_semantic = {str(p) for p in semantic_files} - _stamped_semantic + # #2543: AST failures need both hashes blanked (clear_ast), not just + # semantic_hash — otherwise a prior bad stamp keeps the file "unchanged". + _cleared_ast = set(_failed_ast_sources) + + # Full-scan manifest saves prune rows for in-root files that left the + # scan corpus but still exist on disk (#1908). The corpus must be the + # RAW detect output (files_by_type), NOT the #933-stamp-filtered + # _manifest_files above — pruning to the filtered set would erase + # failed-chunk/omitted-doc rows and every doc row on --code-only runs. + _scan_corpus = ( + {f for _fl in files_by_type.values() for f in _fl} + if has_path else None + ) - def _invalidate_file_manifest_for_db_graph() -> None: - if has_path: - return - try: - manifest_path.unlink(missing_ok=True) - except OSError as exc: - print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) - sys.exit(1) + def _invalidate_file_manifest_for_db_graph() -> None: + if has_path: + return + try: + manifest_path.unlink(missing_ok=True) + except OSError as exc: + print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) + sys.exit(1) - if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - from graphify.export import ( - backup_if_protected as _backup, - existing_graph_node_count as _existing_graph_node_count, - ) - if ( - incremental_mode - and not code_files - and not semantic_files - and not deleted_files - and not pg_result.get("nodes") - and not pg_result.get("edges") - and not cargo_result.get("nodes") - and not cargo_result.get("edges") - ): - # An exclusion-only change reaches this gate (excluded files - # are deliberately NOT in deleted_files, #1908) but must still - # scrub the newly-excluded sources from the raw graph (#1909). - # This path never runs build_merge, so prune in place. - if graph_stale_sources: - _n_pruned = _prune_graph_json_sources( - existing_graph_path, graph_stale_sources + if no_cluster: + # --no-cluster: dump the raw merged extraction as graph.json. + # No NetworkX, no community detection, no analysis sidecar. + # Dedupe nodes (by id) and parallel edges so the raw output matches the + # clustered path (whose DiGraph collapses both) and stays deterministic + # across modes (#1317; node dedup also collapses shared Swift module + # anchors emitted per importing file, #1327). + from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes + from graphify.export import ( + backup_if_protected as _backup, + existing_graph_node_count as _existing_graph_node_count, + ) + if ( + incremental_mode + and not code_files + and not semantic_files + and not deleted_files + and not pg_result.get("nodes") + and not pg_result.get("edges") + and not cargo_result.get("nodes") + and not cargo_result.get("edges") + ): + # An exclusion-only change reaches this gate (excluded files + # are deliberately NOT in deleted_files, #1908) but must still + # scrub the newly-excluded sources from the raw graph (#1909). + # This path never runs build_merge, so prune in place. + if graph_stale_sources: + _n_pruned = _prune_graph_json_sources( + existing_graph_path, graph_stale_sources + ) + if _n_pruned: + print( + f"[graphify extract] pruned {_n_pruned} node(s) from " + f"{len(graph_stale_sources)} source file(s) no longer " + "in the scan (deleted or excluded)." + ) + print( + "[graphify extract] no incremental changes detected " + "(--no-cluster); outputs left untouched." ) - if _n_pruned: + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic, clear_ast=_cleared_ast or None) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + if incremental_mode: + # #2169: this raw path used to write ONLY this run's extraction + # over graph.json — on an incremental run that is just the + # changed files, silently dropping every node/edge owned by an + # unchanged file. Merge the existing graph forward first, with + # the same replace/prune semantics as the clustered path's + # build_merge: re-extracted sources replaced, deleted + + # excluded + graph-stale sources pruned, everything else + # carried. Survivors are prepended, so the dedupe below keeps + # this run's fresh attributes for re-extracted nodes. + from graphify.build import merge_raw_extraction as _merge_raw_extraction + _raw_prune_sources: list[str] = list(deleted_files) + for _src in list(excluded_files) + graph_stale_sources: + if _src not in _raw_prune_sources: + _raw_prune_sources.append(_src) + try: + merged = _merge_raw_extraction( + merged, + graph_path=existing_graph_path, + prune_sources=_raw_prune_sources or None, + root=target, + ) + except RuntimeError as exc: + # Existing graph present but unparseable: refuse to + # raw-dump this run's partial extraction over it. + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + merged["nodes"] = _dedupe_nodes(merged["nodes"]) + merged["edges"] = _dedupe_edges(merged["edges"]) + # Disambiguate colliding-basename file-node labels (#2032). This raw + # --no-cluster path bypasses build_from_json (where the clustered path + # gets this), so apply it directly on the merged node list. + from graphify.build import disambiguate_file_labels_in_nodes as _disamb_labels + _disamb_labels(merged["nodes"]) + # Backfill source_file from endpoint nodes — this raw path bypasses + # build_from_json's backfill, and semantic edges sometimes omit it (#1279). + _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} + for _e in merged["edges"]: + if not _e.get("source_file"): + _e["source_file"] = ( + _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" + ) + # RT-parity for the raw path: an incomplete build must not force a + # partial graph over a larger complete one here either. The clustered + # path gets this from to_json's #479 guard; this path never calls + # to_json, so replicate the shrink check against the existing file and + # exit before the write/manifest unless --allow-partial is set. + if _extraction_incomplete and not cli_allow_partial: + from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH + _existing_n = _existing_graph_node_count(graph_json_path) + _malformed = _existing_n is _MALFORMED_GRAPH + _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n + if _malformed or _shrinks: + _detail = ( + f"the existing {graph_json_path} is present but unparseable " + "(corrupt or a mid-write), so a shrink cannot be ruled out" + if _malformed + else f"smaller than the existing {graph_json_path} " + f"({len(merged['nodes'])} < {_existing_n} nodes)" + ) print( - f"[graphify extract] pruned {_n_pruned} node(s) from " - f"{len(graph_stale_sources)} source file(s) no longer " - "in the scan (deleted or excluded)." + "[graphify extract] error: extraction was incomplete (an AST/" + f"semantic pass failed) and the resulting --no-cluster graph is {_detail}. " + "Refusing to overwrite a complete graph with a partial one. Re-run after " + "fixing the failures, or pass --allow-partial to overwrite anyway.", + file=sys.stderr, ) + sys.exit(1) + _backup(graphify_out) + _invalidate_file_manifest_for_db_graph() + from graphify.paths import write_json_atomic as _write_json_atomic + _write_json_atomic(graph_json_path, merged, indent=2) + try: + # Record the scan root so a later build_merge / update runbook can + # relativize deleted-file paths correctly even for a custom --out + # (its grandparent-of-graph.json fallback points at the wrong dir + # otherwise, and deleted files never prune — #2012/#1571). + (graphify_out / ".graphify_root").write_text( + str(Path(target).resolve()), encoding="utf-8" + ) + except OSError: + pass + stages.mark("write") + cost = _estimate_cost( + backend, merged["input_tokens"], merged["output_tokens"] + ) print( - "[graphify extract] no incremental changes detected " - "(--no-cluster); outputs left untouched." + f"[graphify extract] wrote {graph_json_path} — " + f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " + f"(no clustering)" ) + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost: ${cost:.4f}" + ) try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic, clear_ast=_cleared_ast or None) + if has_path: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic, clear_ast=_cleared_ast or None) except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) stages.total() sys.exit(0) + # Build graph + cluster + score + write. + from graphify.build import ( + build as _build, + build_from_json as _build_from_json, + build_merge as _build_merge, + ) + from graphify.cluster import cluster as _cluster, score_all as _score_all + from graphify.export import to_json as _to_json + from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising + dedup_backend = backend if dedup_llm else None if incremental_mode: - # #2169: this raw path used to write ONLY this run's extraction - # over graph.json — on an incremental run that is just the - # changed files, silently dropping every node/edge owned by an - # unchanged file. Merge the existing graph forward first, with - # the same replace/prune semantics as the clustered path's - # build_merge: re-extracted sources replaced, deleted + - # excluded + graph-stale sources pruned, everything else - # carried. Survivors are prepended, so the dedupe below keeps - # this run's fresh attributes for re-extracted nodes. - from graphify.build import merge_raw_extraction as _merge_raw_extraction - _raw_prune_sources: list[str] = list(deleted_files) + # Prune everything the current scan no longer covers: genuinely + # deleted manifest rows, excluded-but-alive manifest rows (#1908), + # and the graph's own stale sources — which catches files that + # became excluded without ever being manifest-listed (#1909). + _prune_sources: list[str] = list(deleted_files) for _src in list(excluded_files) + graph_stale_sources: - if _src not in _raw_prune_sources: - _raw_prune_sources.append(_src) + if _src not in _prune_sources: + _prune_sources.append(_src) try: - merged = _merge_raw_extraction( - merged, + G = _build_merge( + [merged], graph_path=existing_graph_path, - prune_sources=_raw_prune_sources or None, + prune_sources=_prune_sources or None, + dedup=not no_dedup, + dedup_llm_backend=dedup_backend, root=target, ) - except RuntimeError as exc: - # Existing graph present but unparseable: refuse to - # raw-dump this run's partial extraction over it. - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Disambiguate colliding-basename file-node labels (#2032). This raw - # --no-cluster path bypasses build_from_json (where the clustered path - # gets this), so apply it directly on the merged node list. - from graphify.build import disambiguate_file_labels_in_nodes as _disamb_labels - _disamb_labels(merged["nodes"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) - # RT-parity for the raw path: an incomplete build must not force a - # partial graph over a larger complete one here either. The clustered - # path gets this from to_json's #479 guard; this path never calls - # to_json, so replicate the shrink check against the existing file and - # exit before the write/manifest unless --allow-partial is set. - if _extraction_incomplete and not cli_allow_partial: - from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH - _existing_n = _existing_graph_node_count(graph_json_path) - _malformed = _existing_n is _MALFORMED_GRAPH - _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n - if _malformed or _shrinks: - _detail = ( - f"the existing {graph_json_path} is present but unparseable " - "(corrupt or a mid-write), so a shrink cannot be ruled out" - if _malformed - else f"smaller than the existing {graph_json_path} " - f"({len(merged['nodes'])} < {_existing_n} nodes)" - ) - print( - "[graphify extract] error: extraction was incomplete (an AST/" - f"semantic pass failed) and the resulting --no-cluster graph is {_detail}. " - "Refusing to overwrite a complete graph with a partial one. Re-run after " - "fixing the failures, or pass --allow-partial to overwrite anyway.", - file=sys.stderr, - ) + except ValueError as exc: + # --no-dedup arms build_merge's #479 shrink guard, which refuses + # to drop nodes belonging to files this run neither re-extracted + # nor pruned. Report the refusal instead of a traceback (#2881): + # graph.json on disk is untouched, so the old graph is intact. + print(f"[graphify extract] {exc}", file=sys.stderr) sys.exit(1) + else: + G = _build([merged], dedup=not no_dedup, dedup_llm_backend=dedup_backend, root=target) + stages.mark("build") + if G.number_of_nodes() == 0: + print( + "[graphify extract] graph is empty — extraction produced no nodes. " + "Possible causes: all files skipped, binary-only corpus, or LLM " + "returned no edges.", + file=sys.stderr, + ) + sys.exit(1) + + communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) + stages.mark("cluster") + cohesion = _score_all(G, communities) + try: + gods = _god_nodes(G) + except Exception: + gods = [] + try: + surprises = _surprising(G, communities) + except Exception: + surprises = [] + stages.mark("analyze") + + from graphify.export import backup_if_protected as _backup _backup(graphify_out) _invalidate_file_manifest_for_db_graph() - from graphify.paths import write_json_atomic as _write_json_atomic - _write_json_atomic(graph_json_path, merged, indent=2) + # force=True bypasses the #479 shrink guard entirely. A full build + # legitimately shrinks (fuzzy dedup collapse, deleted code) so it keeps + # force=True — EXCEPT when this run's extraction was incomplete (an + # extractor pass crashed or some semantic chunks failed). Then a partial + # graph could silently overwrite a good complete one, so fall back to the + # shrink guard (force=False) unless the user opts in with --allow-partial. + # + # Both write paths are guarded: the clustered path here via to_json's + # #479 check, and the `--no-cluster` raw-dump path above via the same + # shrink check against the existing file (existing_graph_node_count). + # + # Trade-off: this reuses to_json's coarse node-count guard, not the + # source-aware _check_shrink that watch/update use. On an incremental run + # a legitimate deletion that coincides with an unrelated transient chunk + # failure can therefore be refused here — recoverable by re-running or + # passing --allow-partial (the good graph is preserved and the manifest + # is not stamped, so the retry re-extracts). + _force_write = cli_allow_partial or not _extraction_incomplete + # Stamp provenance from the ANALYSED repo, not the shell's cwd: without + # this, to_json's fallback asks `git rev-parse HEAD` in whatever repo the + # command was invoked from, so `graphify extract ` run from + # another repo's root stamped the invoker's commit into the target's + # graph.json — and cluster then propagates that stamp into + # GRAPH_REPORT.md (#2534 keeps the extract-time stamp by design). Same + # cwd-anchoring mistake #2316 fixed for watch/update, surviving in the + # extract path. + from graphify.watch import _git_head as _gh_target + _wrote = _to_json(G, communities, str(graph_json_path), force=_force_write, + built_at_commit=_gh_target(cwd=Path(target).resolve())) + if not _wrote: + # The shrink guard refused: this partial build is smaller than the + # existing graph. Exit before writing the manifest/marker below, which + # would otherwise stamp these files as done and make the next + # incremental run skip re-extracting them (poisoning the manifest + # against the graph we declined to write). Exit non-zero so a retry + # re-attempts. + print( + "[graphify extract] error: extraction was incomplete (an AST/semantic " + f"pass failed) and the resulting graph is smaller than the existing " + f"{graph_json_path}. Refusing to overwrite a complete graph with a " + "partial one. Re-run after fixing the failures, or pass --allow-partial " + "to overwrite anyway.", + file=sys.stderr, + ) + sys.exit(1) try: - # Record the scan root so a later build_merge / update runbook can - # relativize deleted-file paths correctly even for a custom --out - # (its grandparent-of-graph.json fallback points at the wrong dir - # otherwise, and deleted files never prune — #2012/#1571). + # See the --no-cluster path above: persist the scan root so build_merge + # can relativize deleted-file paths under a custom --out (#2012/#1571). (graphify_out / ".graphify_root").write_text( str(Path(target).resolve()), encoding="utf-8" ) except OSError: pass - stages.mark("write") - cost = _estimate_cost( - backend, merged["input_tokens"], merged["output_tokens"] - ) - print( - f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " - f"(no clustering)" - ) - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost: ${cost:.4f}" + stages.mark("export") + if merged.get("output_tokens", 0) > 0: + (graphify_out / ".graphify_semantic_marker").write_text( + json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" ) - try: - if has_path: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic, clear_ast=_cleared_ast or None) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) if global_merge: from graphify.global_graph import global_add as _global_add _tag = global_repo_tag or target.name @@ -3943,193 +4255,57 @@ def _invalidate_file_manifest_for_db_graph() -> None: f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") except Exception as exc: print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - # Build graph + cluster + score + write. - from graphify.build import ( - build as _build, - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - dedup_backend = backend if dedup_llm else None - if incremental_mode: - # Prune everything the current scan no longer covers: genuinely - # deleted manifest rows, excluded-but-alive manifest rows (#1908), - # and the graph's own stale sources — which catches files that - # became excluded without ever being manifest-listed (#1909). - _prune_sources: list[str] = list(deleted_files) - for _src in list(excluded_files) + graph_stale_sources: - if _src not in _prune_sources: - _prune_sources.append(_src) - try: - G = _build_merge( - [merged], - graph_path=existing_graph_path, - prune_sources=_prune_sources or None, - dedup=not no_dedup, - dedup_llm_backend=dedup_backend, - root=target, - ) - except ValueError as exc: - # --no-dedup arms build_merge's #479 shrink guard, which refuses - # to drop nodes belonging to files this run neither re-extracted - # nor pruned. Report the refusal instead of a traceback (#2881): - # graph.json on disk is untouched, so the old graph is intact. - print(f"[graphify extract] {exc}", file=sys.stderr) - sys.exit(1) - else: - G = _build([merged], dedup=not no_dedup, dedup_llm_backend=dedup_backend, root=target) - stages.mark("build") - if G.number_of_nodes() == 0: - print( - "[graphify extract] graph is empty — extraction produced no nodes. " - "Possible causes: all files skipped, binary-only corpus, or LLM " - "returned no edges.", - file=sys.stderr, - ) - sys.exit(1) - - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) - stages.mark("cluster") - cohesion = _score_all(G, communities) - try: - gods = _god_nodes(G) - except Exception: - gods = [] - try: - surprises = _surprising(G, communities) - except Exception: - surprises = [] - stages.mark("analyze") - - from graphify.export import backup_if_protected as _backup - _backup(graphify_out) - _invalidate_file_manifest_for_db_graph() - # force=True bypasses the #479 shrink guard entirely. A full build - # legitimately shrinks (fuzzy dedup collapse, deleted code) so it keeps - # force=True — EXCEPT when this run's extraction was incomplete (an - # extractor pass crashed or some semantic chunks failed). Then a partial - # graph could silently overwrite a good complete one, so fall back to the - # shrink guard (force=False) unless the user opts in with --allow-partial. - # - # Both write paths are guarded: the clustered path here via to_json's - # #479 check, and the `--no-cluster` raw-dump path above via the same - # shrink check against the existing file (existing_graph_node_count). - # - # Trade-off: this reuses to_json's coarse node-count guard, not the - # source-aware _check_shrink that watch/update use. On an incremental run - # a legitimate deletion that coincides with an unrelated transient chunk - # failure can therefore be refused here — recoverable by re-running or - # passing --allow-partial (the good graph is preserved and the manifest - # is not stamped, so the retry re-extracts). - _force_write = cli_allow_partial or not _extraction_incomplete - # Stamp provenance from the ANALYSED repo, not the shell's cwd: without - # this, to_json's fallback asks `git rev-parse HEAD` in whatever repo the - # command was invoked from, so `graphify extract ` run from - # another repo's root stamped the invoker's commit into the target's - # graph.json — and cluster then propagates that stamp into - # GRAPH_REPORT.md (#2534 keeps the extract-time stamp by design). Same - # cwd-anchoring mistake #2316 fixed for watch/update, surviving in the - # extract path. - from graphify.watch import _git_head as _gh_target - _wrote = _to_json(G, communities, str(graph_json_path), force=_force_write, - built_at_commit=_gh_target(cwd=Path(target).resolve())) - if not _wrote: - # The shrink guard refused: this partial build is smaller than the - # existing graph. Exit before writing the manifest/marker below, which - # would otherwise stamp these files as done and make the next - # incremental run skip re-extracting them (poisoning the manifest - # against the graph we declined to write). Exit non-zero so a retry - # re-attempts. - print( - "[graphify extract] error: extraction was incomplete (an AST/semantic " - f"pass failed) and the resulting graph is smaller than the existing " - f"{graph_json_path}. Refusing to overwrite a complete graph with a " - "partial one. Re-run after fixing the failures, or pass --allow-partial " - "to overwrite anyway.", - file=sys.stderr, - ) - sys.exit(1) - try: - # See the --no-cluster path above: persist the scan root so build_merge - # can relativize deleted-file paths under a custom --out (#2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" - ) - except OSError: - pass - stages.mark("export") - if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" - ) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "tokens": { + "input": merged["input_tokens"], + "output": merged["output_tokens"], + }, + } + from graphify.paths import write_json_atomic as _wja + _wja(analysis_path, analysis, indent=2) try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + if has_path: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic, clear_ast=_cleared_ast or None) except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "tokens": { - "input": merged["input_tokens"], - "output": merged["output_tokens"], - }, - } - from graphify.paths import write_json_atomic as _wja - _wja(analysis_path, analysis, indent=2) - try: - if has_path: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic, clear_ast=_cleared_ast or None) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" + cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted{_excl_note}" + f"[graphify extract] wrote {graph_json_path}: " + f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " + f"{len(communities)} communities" ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: + print(f"[graphify extract] wrote {analysis_path}") + if incremental_mode: + _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" + print( + f"[graphify extract] incremental summary: " + f"{sem_cache_hits + unchanged_total} files cached/unchanged, " + f"{len(code_files) + sem_cache_misses} re-extracted, " + f"{len(deleted_files)} deleted{_excl_note}" + ) + elif sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost (~{backend}): ${cost:.4f}" + ) + # extract intentionally stops at graph.json + analysis; the report and + # community labels are produced by `cluster-only` (or an agent's Step 5). + # Point standalone users at it so communities get named (#1097). print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" + "[graphify extract] next: run " + f"`graphify cluster-only {graphify_out.parent}` " + "to generate GRAPH_REPORT.md and name communities" ) - # extract intentionally stops at graph.json + analysis; the report and - # community labels are produced by `cluster-only` (or an agent's Step 5). - # Point standalone users at it so communities get named (#1097). - print( - "[graphify extract] next: run " - f"`graphify cluster-only {graphify_out.parent}` " - "to generate GRAPH_REPORT.md and name communities" - ) - stages.total() + stages.total() elif cmd == "cache-check": # graphify cache-check [--root ] [--mode | --deep] diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py index 14c47f0d5..b77c9a901 100644 --- a/graphify/exporters/graphdb.py +++ b/graphify/exporters/graphdb.py @@ -6,12 +6,20 @@ import re +def _batch_rows(rows: list[dict], batch_size: int): + """Yield ``rows`` in chunks of at most ``batch_size``.""" + for start in range(0, len(rows), batch_size): + yield rows[start:start + batch_size] + + def push_to_neo4j( G: nx.Graph, uri: str, user: str, password: str, communities: dict[int, list[str]] | None = None, + *, + batch_size: int = 100, ) -> dict[str, int]: """Push graph directly to a running Neo4j instance via the Python driver. @@ -19,6 +27,14 @@ def push_to_neo4j( Uses MERGE so re-running is safe - nodes and edges are upserted, not duplicated. Returns a dict with counts of nodes and edges pushed. + + Rows are sent in UNWIND batches of ``batch_size`` (default 100) instead of + one query per node/edge - a per-entry push spends nearly all its time on + round trips once the server is not on localhost. Node labels and + relationship types are baked into the Cypher text (they cannot be + parameters), so rows are grouped by sanitized label/relation first and each + group is batched separately. UNWIND processes rows in order, so duplicates + inside one batch upsert exactly as the per-entry queries did. """ try: from neo4j import GraphDatabase @@ -27,6 +43,9 @@ def push_to_neo4j( "neo4j driver not installed. Run: pip install neo4j" ) from e + if batch_size < 1: + raise ValueError(f"batch_size must be a positive integer, got {batch_size}") + node_community = _node_community_map(communities) if communities else {} def _safe_rel(relation: str) -> str: @@ -37,42 +56,51 @@ def _safe_label(label: str) -> str: sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) return sanitized if sanitized else "Entity" + node_rows: dict[str, list[dict]] = {} + for node_id, data in G.nodes(data=True): + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + props["id"] = node_id + cid = node_community.get(node_id) + if cid is not None: + props["community"] = cid + ftype = _safe_label(data.get("file_type", "Entity").capitalize()) + node_rows.setdefault(ftype, []).append({"id": node_id, "props": props}) + + edge_rows: dict[str, list[dict]] = {} + for u, v, data in G.edges(data=True): + rel = _safe_rel(data.get("relation", "RELATED_TO")) + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + edge_rows.setdefault(rel, []).append({"src": u, "tgt": v, "props": props}) + driver = GraphDatabase.driver(uri, auth=(user, password)) nodes_pushed = 0 edges_pushed = 0 with driver.session() as session: - for node_id, data in G.nodes(data=True): - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - props["id"] = node_id - cid = node_community.get(node_id) - if cid is not None: - props["community"] = cid - ftype = _safe_label(data.get("file_type", "Entity").capitalize()) - session.run( - f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", - id=node_id, - props=props, - ) - nodes_pushed += 1 - - for u, v, data in G.edges(data=True): - rel = _safe_rel(data.get("relation", "RELATED_TO")) - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - session.run( - f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - src=u, - tgt=v, - props=props, - ) - edges_pushed += 1 + for ftype, rows in node_rows.items(): + for batch in _batch_rows(rows, batch_size): + session.run( + f"UNWIND $rows AS row " + f"MERGE (n:{ftype} {{id: row.id}}) SET n += row.props", + rows=batch, + ) + nodes_pushed += len(batch) + + for rel, rows in edge_rows.items(): + for batch in _batch_rows(rows, batch_size): + session.run( + f"UNWIND $rows AS row " + f"MATCH (a {{id: row.src}}), (b {{id: row.tgt}}) " + f"MERGE (a)-[r:{rel}]->(b) SET r += row.props", + rows=batch, + ) + edges_pushed += len(batch) driver.close() return {"nodes": nodes_pushed, "edges": edges_pushed} @@ -84,13 +112,18 @@ def push_to_falkordb( password: str | None = None, communities: dict[int, list[str]] | None = None, graph_name: str = "graphify", + *, + batch_size: int = 100, ) -> dict[str, int]: """Push graph directly to a running FalkorDB instance via the Python SDK. Requires: pip install falkordb FalkorDB is OpenCypher-compatible, so the MERGE/SET upsert queries are - identical to push_to_neo4j. Differences from the Neo4j path: + identical to push_to_neo4j - including the UNWIND batching (``batch_size`` + rows per round trip, grouped by sanitized label/relation because those are + baked into the Cypher text and cannot be parameters). Differences from the + Neo4j path: - connects with FalkorDB(host, port, username, password) instead of a bolt driver; only the host/port are read from the URI, so the scheme is informational - "falkordb://localhost:6379", "redis://localhost:6379" @@ -112,6 +145,9 @@ def push_to_falkordb( "falkordb SDK not installed. Run: pip install falkordb" ) from e + if batch_size < 1: + raise ValueError(f"batch_size must be a positive integer, got {batch_size}") + from urllib.parse import urlparse node_community = _node_community_map(communities) if communities else {} @@ -137,10 +173,7 @@ def _safe_label(label: str) -> str: username=connect_user, password=connect_password, ) - graph = db.select_graph(graph_name) - nodes_pushed = 0 - edges_pushed = 0 - + node_rows: dict[str, list[dict]] = {} for node_id, data in G.nodes(data=True): props = { k: v for k, v in data.items() @@ -151,23 +184,38 @@ def _safe_label(label: str) -> str: if cid is not None: props["community"] = cid ftype = _safe_label(data.get("file_type", "Entity").capitalize()) - graph.query( - f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", - {"id": node_id, "props": props}, - ) - nodes_pushed += 1 + node_rows.setdefault(ftype, []).append({"id": node_id, "props": props}) + edge_rows: dict[str, list[dict]] = {} for u, v, data in G.edges(data=True): rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") } - graph.query( - f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - {"src": u, "tgt": v, "props": props}, - ) - edges_pushed += 1 + edge_rows.setdefault(rel, []).append({"src": u, "tgt": v, "props": props}) + + graph = db.select_graph(graph_name) + nodes_pushed = 0 + edges_pushed = 0 + + for ftype, rows in node_rows.items(): + for batch in _batch_rows(rows, batch_size): + graph.query( + f"UNWIND $rows AS row " + f"MERGE (n:{ftype} {{id: row.id}}) SET n += row.props", + {"rows": batch}, + ) + nodes_pushed += len(batch) + + for rel, rows in edge_rows.items(): + for batch in _batch_rows(rows, batch_size): + graph.query( + f"UNWIND $rows AS row " + f"MATCH (a {{id: row.src}}), (b {{id: row.tgt}}) " + f"MERGE (a)-[r:{rel}]->(b) SET r += row.props", + {"rows": batch}, + ) + edges_pushed += len(batch) return {"nodes": nodes_pushed, "edges": edges_pushed} diff --git a/graphify/llm.py b/graphify/llm.py index 5e410af21..fab4f2f30 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -123,6 +123,12 @@ def _resolve_ollama_base_url(default: str) -> str: "pricing": {"input": 0.74, "output": 4.66}, # USD per 1M tokens "temperature": None, # kimi-k2.6 enforces its own fixed temperature; sending any value raises 400 "max_tokens": 16384, + # Reasoning effort for Kimi models that support it (K3 advertises + # valid_efforts ["low","high","max"] on /models). Upstream previously sent + # nothing, so the server default ("high" for K3) silently applied while the + # gemini block above has an effort setting. Env-overridable; forwarded as + # `reasoning_effort` by the existing plumbing. + "reasoning_effort": os.environ.get("GRAPHIFY_KIMI_EFFORT", "max"), }, "ollama": { "base_url": _resolve_ollama_base_url("http://localhost:11434/v1"), @@ -216,6 +222,19 @@ def _resolve_ollama_base_url(default: str) -> str: # CLI's Read tool rather than as inline base64 (see `_call_claude_cli`). "vision": True, }, + # PATCHED FOR TELB-COCKPIT (2026-08-19). This explicit backend routes graph + # extraction through the owner's authenticated Codex CLI/ChatGPT subscription. + # If an upgrade removes it, the backend silently disappears and extraction can + # fall back to the metered OpenAI API — a spending regression. + "openai-cli": { + "default_model": "gpt-5.6-sol", + "model_env_key": "GRAPHIFY_OPENAI_CLI_MODEL", + # Subscription usage is not metered API spend; zero pricing is intentional. + "pricing": {"input": 0.0, "output": 0.0}, + "temperature": None, + "max_tokens": 16384, + "vision": False, + }, } @@ -1628,6 +1647,52 @@ def _claude_cli_supports_json_schema(claude_cmd: str) -> bool: return supported +def _openai_cli_turn_usage(stdout: str) -> tuple[int, int]: + """Return token counts from the last Codex ``turn.completed`` JSONL event.""" + completed_usage: dict | None = None + for line in (stdout or "").splitlines(): + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except (json.JSONDecodeError, TypeError): + # Codex stdout is JSONL, but tolerate diagnostics or malformed lines; + # losing accounting must never discard an otherwise valid extraction. + continue + if isinstance(event, dict) and event.get("type") == "turn.completed": + usage = event.get("usage") + completed_usage = usage if isinstance(usage, dict) else {} + + if completed_usage is None: + # Without a completion event the exact counts are unknown; report zero + # rather than estimating or inventing usage. + return 0, 0 + + def _count(name: str) -> int: + try: + return int(completed_usage.get(name, 0) or 0) + except (TypeError, ValueError): + return 0 + + # Codex input_tokens already appears to include cached_input_tokens, so do + # not add the cached count again or usage will be double-counted. + return _count("input_tokens"), _count("output_tokens") + + +def _openai_cli_vendor_detail(stderr: str, stdout: str) -> str: + """Return bounded Codex diagnostics, preserving stderr and stdout's tail.""" + parts: list[str] = [] + stderr_text = (stderr or "").strip() + stdout_text = (stdout or "").strip() + if stderr_text: + parts.append(f"stderr: {stderr_text[-400:]}") + if stdout_text: + # API errors, including HTTP 400 responses, can appear only in JSONL stdout. + parts.append(f"stdout tail: {stdout_text[-400:]}") + return " | ".join(parts) or "(no stderr or stdout)" + + def _call_claude_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bool = False, images: list[_ImageRef] | None = None) -> dict: """Call Claude via the locally-installed Claude Code CLI (`claude -p`). @@ -1774,6 +1839,160 @@ def _call_claude_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bo return result +def _codex_disable_mcp_args(codex_cmd: str) -> list[str]: + """`-c mcp_servers..enabled=false` for every server Codex has configured. + + Extraction never calls a tool, and each configured MCP server is started per + `codex exec` invocation. Codex config overrides deep-merge, so a blanket + `-c mcp_servers={}` leaves the servers enabled; its per-server `enabled` field + (visible in `codex mcp get `) is the switch that works. The server list + comes from `codex mcp list --json`, so no server name is hardcoded. Best effort: + if that call is unavailable (older Codex, no config), extraction proceeds with + whatever the user configured. + """ + import subprocess + + try: + proc = subprocess.run( + [codex_cmd, "mcp", "list", "--json"], + capture_output=True, text=True, encoding="utf-8", errors="replace", + timeout=30, check=False, **_no_window_kwargs(), + ) + if proc.returncode != 0 or not proc.stdout.strip(): + return [] + servers = json.loads(proc.stdout) + except Exception: + return [] + args: list[str] = [] + for entry in servers if isinstance(servers, list) else []: + name = (entry or {}).get("name") if isinstance(entry, dict) else None + if isinstance(name, str) and name and all(ch.isalnum() or ch in "-_" for ch in name): + args += ["-c", f"mcp_servers.{name}.enabled=false"] + return args + + +def _call_openai_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bool = False, images: list[_ImageRef] | None = None) -> dict: + """Call OpenAI through the locally authenticated Codex CLI.""" + # PATCHED FOR TELB-COCKPIT (2026-08-19). This keeps semantic extraction on + # the owner's ChatGPT OAuth subscription. Reverting it removes the safe CLI + # route and can send the same work through a metered OPENAI_API_KEY instead. + import shutil + import subprocess + import tempfile + + codex_cmd = shutil.which("codex") + if codex_cmd is None: + raise RuntimeError( + "OpenAI Codex CLI not found on $PATH. Install it with " + "`npm install -g @openai/codex` and run `codex` once to authenticate " + "with your ChatGPT account." + ) + + mdl = os.environ.get("GRAPHIFY_OPENAI_CLI_MODEL", "").strip() or "gpt-5.6-sol" + if images: + # This backend is text-only: preserve image source references without + # exposing absolute paths or claiming that Codex received pixel data. + user_message = _with_image_notes(user_message, images, with_paths=False) + + # Deliver the schema and imperative together in the CLI's user turn, just as + # claude-cli does. An agentic CLI given only raw source may answer in prose; + # that would parse hollow and trigger the adaptive retry path. + combined_message = ( + _extraction_system(deep=deep_mode) + + "\n\n" + + "Now extract the knowledge graph from the following source file(s) " + + "and output ONLY the JSON object described above. No prose, no " + + "preamble, no markdown fences.\n\n" + + user_message + ) + + output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".json") + output_path = output_file.name + output_file.close() + try: + # Linux MAX_ARG_STRLEN caps a single argv entry at 128 KB. Real chunks + # reach 240-306 KB, well past that cap; putting the prompt in argv raises + # Errno 7 in subprocess before Codex starts, so there are no vendor + # diagnostics at all. + cli_args = [ + codex_cmd, + "exec", + "--skip-git-repo-check", + "--json", + # Codex can execute shell commands. Extraction must not be able to + # write to the corpus it is reading, so the sandbox is read-only. + "--sandbox", + "read-only", + # Extraction is text-to-JSON: it asks no questions of any tool. Without + # this, every `codex exec` starts the user's configured MCP servers per + # call (measured: four graph servers at ~152 MB each during one run, + # more memory than the extraction itself). `-c mcp_servers={}` does NOT + # work — Codex merges config overrides into the table rather than + # replacing it, so the servers stay enabled (verified with + # `codex mcp list -c mcp_servers={}`). The mechanism that does work is + # Codex's own per-server `enabled` field, one override per server. + *_codex_disable_mcp_args(codex_cmd), + # Reasoning effort for the extraction calls (owner's setting; the env + # var overrides without a source edit, mirroring GRAPHIFY_KIMI_EFFORT). + "-c", + "model_reasoning_effort=%s" % (os.environ.get("GRAPHIFY_OPENAI_CLI_EFFORT", "").strip() or "ultra"), + "--model", + mdl, + "-o", + output_path, + "-", + ] + proc = subprocess.run( + cli_args, + # input= keeps the prompt out of argv and closes the pipe; the prompt + # cannot ride in argv, and an open harness pipe hangs Codex forever. + input=combined_message, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + # A measured single-document extraction took 361 seconds, so the + # 600-second default is tight; users should raise --api-timeout. + timeout=_resolve_api_timeout(), + check=False, + **_no_window_kwargs(), + ) + if proc.returncode != 0: + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec exited {proc.returncode}: {detail}") + if not os.path.exists(output_path): + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec produced no -o output file: {detail}") + try: + raw_content = Path(output_path).read_text(encoding="utf-8", errors="replace") + except OSError as exc: + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec could not read its -o output: {exc}; {detail}") from exc + if not raw_content.strip(): + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec produced an empty -o output file: {detail}") + + result = _parse_llm_json(raw_content) + if not result.get("nodes") and not result.get("edges") and not result.get("hyperedges"): + # Never return an empty graph here: _response_is_hollow would treat it + # as truncation and bisect the chunk into as many as 15 more calls. That + # exact failure mode previously consumed 87% of an hourly backend quota. + vendor_text = raw_content.strip()[:800] + raise RuntimeError(f"codex exec returned no graph content: {vendor_text}") + + input_tokens, output_tokens = _openai_cli_turn_usage(proc.stdout) + result["input_tokens"] = input_tokens + result["output_tokens"] = output_tokens + result["model"] = mdl + result["finish_reason"] = "stop" + return result + finally: + try: + os.unlink(output_path) + except OSError: + pass + + def _azure_client(api_key: str, endpoint: str): """Construct an AzureOpenAI client with env-driven api_version and timeout.""" try: @@ -1931,7 +2150,7 @@ def extract_files_direct( file=sys.stderr, ) key = "ollama" - if not key and backend not in ("bedrock", "claude-cli"): + if not key and backend not in ("bedrock", "claude-cli", "openai-cli"): raise ValueError( f"No API key for backend '{backend}'. " f"Set {_format_backend_env_keys(backend)} or pass api_key=." @@ -1955,6 +2174,8 @@ def extract_files_direct( result = _call_claude(key, mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) elif backend == "claude-cli": result = _call_claude_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + elif backend == "openai-cli": + result = _call_openai_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) elif backend == "bedrock": result = _call_bedrock(mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) elif backend == "azure": @@ -2568,6 +2789,8 @@ def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | # over session state. Force serial unless the user explicitly opts in. if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1": max_concurrency = 1 + if backend == "openai-cli" and os.environ.get("GRAPHIFY_OPENAI_CLI_PARALLEL", "").strip() != "1": + max_concurrency = 1 def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # Persist each chunk's semantic results to the cache as soon as it # completes. Without this, the semantic cache is only written once, at @@ -2810,7 +3033,7 @@ def _call_llm( ollama_url = _resolve_ollama_base_url(cfg.get("base_url", "")) _validate_ollama_base_url(ollama_url) key = "ollama" - if not key and backend not in ("bedrock", "claude-cli"): + if not key and backend not in ("bedrock", "claude-cli", "openai-cli"): raise ValueError( f"No API key for backend '{backend}'. Set {_format_backend_env_keys(backend)}." ) @@ -2884,6 +3107,71 @@ def _rec(inp, out) -> None: ) return envelope.get("result", "") + if backend == "openai-cli": + import shutil, subprocess, tempfile + + codex_cmd = shutil.which("codex") + if codex_cmd is None: + raise RuntimeError( + "OpenAI Codex CLI not found on $PATH. Install it with " + "`npm install -g @openai/codex` and run `codex` once to authenticate " + "with your ChatGPT account." + ) + output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".json") + output_path = output_file.name + output_file.close() + try: + cli_args = [ + codex_cmd, + "exec", + "--skip-git-repo-check", + "--json", + # Labeling is also an agentic Codex pass, so it must not be able + # to mutate the corpus or any other file in the working tree. + "--sandbox", + "read-only", + "--model", + mdl, + "-o", + output_path, + "-", + ] + proc = subprocess.run( + cli_args, + # input= keeps the prompt out of argv and closes the pipe; the prompt + # cannot ride in argv, and an open harness pipe hangs Codex forever. + input=prompt, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=_resolve_api_timeout(), + check=False, + **_no_window_kwargs(), + ) + if proc.returncode != 0: + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec exited {proc.returncode}: {detail}") + if not os.path.exists(output_path): + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec produced no -o output file: {detail}") + try: + raw_content = Path(output_path).read_text(encoding="utf-8", errors="replace") + except OSError as exc: + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec could not read its -o output: {exc}; {detail}") from exc + if not raw_content.strip(): + detail = _openai_cli_vendor_detail(proc.stderr, proc.stdout) + raise RuntimeError(f"codex exec produced an empty -o output file: {detail}") + input_tokens, output_tokens = _openai_cli_turn_usage(proc.stdout) + _rec(input_tokens, output_tokens) + return raw_content + finally: + try: + os.unlink(output_path) + except OSError: + pass + if backend == "bedrock": try: @@ -3082,7 +3370,7 @@ def detect_backend() -> str | None: _validate_ollama_base_url(ollama_url) return "ollama" for name in BACKENDS: - if name not in ("gemini", "kimi", "claude", "openai", "deepseek", "azure", "bedrock", "ollama", "claude-cli"): + if name not in ("gemini", "kimi", "claude", "openai", "deepseek", "azure", "bedrock", "ollama", "claude-cli", "openai-cli"): if _get_backend_api_key(name): return name return None @@ -3300,6 +3588,8 @@ def label_communities( max_concurrency = 1 if backend == "claude-cli" and os.environ.get("GRAPHIFY_CLAUDE_CLI_PARALLEL", "").strip() != "1": max_concurrency = 1 + if backend == "openai-cli" and os.environ.get("GRAPHIFY_OPENAI_CLI_PARALLEL", "").strip() != "1": + max_concurrency = 1 workers = max(1, min(max_concurrency, n_batches)) def _run_batch(batch_idx: int): diff --git a/graphify/watch.py b/graphify/watch.py index 8ad02c4df..ce1a0fe77 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -156,13 +156,20 @@ def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: @contextlib.contextmanager -def _rebuild_lock(out_dir: Path, *, blocking: bool = False): +def _rebuild_lock(out_dir: Path, *, blocking: bool = False, timeout: float | None = None): """Per-repo advisory lock around a rebuild. Yields True if acquired, False if another rebuild is already running and ``blocking`` is False. Uses fcntl.flock so the lock is released automatically if the process is killed (no stale-lock cleanup needed). + ``timeout`` (seconds) only applies when ``blocking`` is True. The kernel + offers just two modes — skip (LOCK_NB) or wait forever (LOCK_EX) — so a + bounded wait is done in userspace: non-blocking attempts with short + sleeps until the deadline, then yield False. Callers that must not hang + indefinitely behind a wedged rebuild (the headless ``extract`` CLI) pass + a timeout so they can report the holder and exit instead. + While the lock is held, ``.rebuild.lock`` contains the owning PID followed by a newline so external pollers (publish scripts, etc.) can read it. On successful release the file is unlinked so downstream tooling that @@ -184,13 +191,25 @@ def _rebuild_lock(out_dir: Path, *, blocking: bool = False): fh = open(lock_path, "a+", encoding="utf-8") acquired = False try: - flags = fcntl.LOCK_EX if blocking else (fcntl.LOCK_EX | fcntl.LOCK_NB) - try: - fcntl.flock(fh.fileno(), flags) - except BlockingIOError: - yield False - return - acquired = True + if blocking and timeout is not None: + deadline = time.monotonic() + timeout + while not acquired: + try: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + except BlockingIOError: + if time.monotonic() >= deadline: + yield False + return + time.sleep(0.2) + else: + flags = fcntl.LOCK_EX if blocking else (fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + fcntl.flock(fh.fileno(), flags) + except BlockingIOError: + yield False + return + acquired = True # Replace any prior owner's PID with ours so external readers see a # single parseable line, not a digit-concatenation across rebuilds. try: @@ -1912,6 +1931,51 @@ def _notify_only(watch_path: Path) -> None: print(f"[graphify watch] Flag written to {flag}") +def _run_semantic_extract( + watch_path: Path, + *, + backend: str | None = None, + fallback_backend: str | None = None, +) -> bool: + """Run LLM-backed semantic extraction over ``watch_path`` in a subprocess. + + A subprocess, not an in-process call, on purpose: ``fcntl.flock`` is held + per open file description, so re-entering ``_rebuild_lock`` from the + watcher's own process would block against itself forever. A child process + contends on ``graphify-out/.rebuild.lock`` like any other invoker, which + is exactly the serialization we want against hook-driven rebuilds. + + On success the ``needs_update`` flag is cleared here: extract never + touches the flag (only ``_rebuild_code`` does), so without this a + successful semantic run would leave a stale "run /graphify --update" + prompt behind. On any failure the flag is left for the caller to raise + via ``_notify_only`` — the user must never lose the manual instruction. + + Returns True when the extract subprocess completed successfully. + """ + import subprocess as _sp + + cmd = [sys.executable, "-m", "graphify", "extract", str(watch_path)] + if backend: + cmd += ["--backend", backend] + if fallback_backend: + cmd += ["--fallback-backend", fallback_backend] + print(f"\n[graphify watch] Non-code files changed - running semantic extraction...") + try: + rc = _sp.run(cmd).returncode + except OSError as exc: + print(f"[graphify watch] Semantic extraction failed to start: {exc}") + return False + if rc != 0: + print(f"[graphify watch] Semantic extraction exited with code {rc}") + return False + flag = watch_path / _GRAPHIFY_OUT / "needs_update" + if flag.exists(): + flag.unlink() + print("[graphify watch] Semantic extraction complete.") + return True + + def _has_non_code(changed_paths: list[Path]) -> bool: return any(p.suffix.lower() not in _CODE_EXTENSIONS for p in changed_paths) @@ -1940,7 +2004,14 @@ def _batch_needs_llm_flag(batch: list[Path]) -> bool: return _has_non_code([p for p in batch if p.exists()]) -def watch(watch_path: Path, debounce: float = 3.0) -> None: +def watch( + watch_path: Path, + debounce: float = 3.0, + *, + semantic: bool = False, + backend: str | None = None, + fallback_backend: str | None = None, +) -> None: """ Watch watch_path for new or modified files and auto-update the graph. @@ -1950,6 +2021,17 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: debounce: seconds to wait after the last change before triggering (avoids running on every keystroke when many files are saved at once). + + semantic: run the LLM-backed extract automatically (as a subprocess — see + ``_run_semantic_extract`` for why in-process re-entry is forbidden) instead + of only writing the needs_update flag. The extract runs synchronously; the + observer thread keeps queueing events meanwhile, so nothing is lost — the + next debounce window picks them up. If the extract fails, the flag + + manual instruction are emitted exactly as without ``semantic``. + + backend / fallback_backend: forwarded to the extract subprocess + (``--backend`` / ``--fallback-backend``) when set; only meaningful with + ``semantic=True``. """ try: from watchdog.observers import Observer @@ -2008,8 +2090,12 @@ def on_any_event(self, event): observer.start() print(f"[graphify watch] Watching {watch_path.resolve()} - press Ctrl+C to stop") - print(f"[graphify watch] Code changes rebuild graph automatically. " - f"Doc/image changes require /graphify --update.") + if semantic: + print(f"[graphify watch] Code changes rebuild graph automatically. " + f"Doc/image changes run semantic extraction automatically (--semantic).") + else: + print(f"[graphify watch] Code changes rebuild graph automatically. " + f"Doc/image changes require /graphify --update.") print(f"[graphify watch] Debounce: {debounce}s") try: @@ -2023,7 +2109,19 @@ def on_any_event(self, event): if _batch_triggers_rebuild(batch): _rebuild_code(watch_path) if _batch_needs_llm_flag(batch): - _notify_only(watch_path) + extracted = False + if semantic: + # At this point the watcher holds no rebuild lock — + # _rebuild_code above acquired and released it + # internally — so the extract subprocess can take + # .rebuild.lock without deadlocking on us. + extracted = _run_semantic_extract( + watch_path, + backend=backend, + fallback_backend=fallback_backend, + ) + if not extracted: + _notify_only(watch_path) except KeyboardInterrupt: print("\n[graphify watch] Stopped.") finally: @@ -2037,5 +2135,12 @@ def on_any_event(self, event): parser.add_argument("path", nargs="?", default=".", help="Folder to watch (default: .)") parser.add_argument("--debounce", type=float, default=3.0, help="Seconds to wait after last change before updating (default: 3)") + parser.add_argument("--semantic", action="store_true", + help="Run LLM-backed semantic extraction automatically on doc/image changes") + parser.add_argument("--backend", default=None, + help="Extraction backend for --semantic (passed to graphify extract)") + parser.add_argument("--fallback-backend", default=None, + help="Fallback extraction backend for --semantic (passed to graphify extract)") args = parser.parse_args() - watch(Path(args.path), debounce=args.debounce) + watch(Path(args.path), debounce=args.debounce, semantic=args.semantic, + backend=args.backend, fallback_backend=args.fallback_backend) diff --git a/tests/test_batched_push.py b/tests/test_batched_push.py new file mode 100644 index 000000000..8fb51396d --- /dev/null +++ b/tests/test_batched_push.py @@ -0,0 +1,242 @@ +"""Unit tests for the batched UNWIND push to Neo4j and FalkorDB. + +The per-entry push ran one query per node/edge, so a remote push spent nearly +all its time on round trips. These tests verify the batched replacement with +fake in-memory drivers (no server, no network): rows are grouped by sanitized +label/relation (labels cannot be Cypher parameters), chunked into batch_size +rows per query, and the row payloads carry exactly the per-entry params so +MERGE/SET upsert semantics are unchanged. +""" +from __future__ import annotations + +import subprocess +import sys +import types +from pathlib import Path + +import networkx as nx +import pytest + +PYTHON = sys.executable + + +# --------------------------------------------------------------------------- +# Fake drivers - record every (cypher, params) pair, touch no network. +# --------------------------------------------------------------------------- + +def _fake_neo4j_module(recorded: list) -> types.ModuleType: + """A stand-in for the `neo4j` package recording every session.run call.""" + + class _Session: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def run(self, cypher, **params): + recorded.append((cypher, params)) + + class _Driver: + def session(self): + return _Session() + + def close(self): + pass + + class GraphDatabase: + @staticmethod + def driver(uri, auth): + return _Driver() + + mod = types.ModuleType("neo4j") + mod.GraphDatabase = GraphDatabase + return mod + + +def _fake_falkordb_module(recorded: list) -> types.ModuleType: + """A stand-in for the `falkordb` package recording every graph.query call.""" + + class _Graph: + def query(self, cypher, params): + recorded.append((cypher, params)) + + class FalkorDB: + def __init__(self, host, port, username=None, password=None): + pass + + def select_graph(self, name): + return _Graph() + + mod = types.ModuleType("falkordb") + mod.FalkorDB = FalkorDB + return mod + + +def _make_graph(n_nodes: int = 0) -> nx.MultiDiGraph: + G = nx.MultiDiGraph() + for i in range(n_nodes): + G.add_node(f"node-{i}", file_type="python", label=f"Node {i}") + return G + + +# --------------------------------------------------------------------------- +# Neo4j +# --------------------------------------------------------------------------- + +def test_neo4j_nodes_are_chunked_by_batch_size(monkeypatch): + recorded: list = [] + monkeypatch.setitem(sys.modules, "neo4j", _fake_neo4j_module(recorded)) + from graphify.export import push_to_neo4j + + G = _make_graph(250) + result = push_to_neo4j(G, uri="bolt://x", user="neo4j", password="pw", + batch_size=100) + + assert result == {"nodes": 250, "edges": 0} + # One label -> ceil(250 / 100) = 3 round trips instead of 250. + assert len(recorded) == 3 + assert [len(p["rows"]) for _, p in recorded] == [100, 100, 50] + for cypher, _ in recorded: + assert "UNWIND $rows AS row" in cypher + assert "MERGE (n:Python {id: row.id})" in cypher + assert "SET n += row.props" in cypher + # Every node arrives exactly once, in graph order, with the id in props + # exactly as the per-entry queries sent it. + all_rows = [r for _, p in recorded for r in p["rows"]] + assert [r["id"] for r in all_rows] == [f"node-{i}" for i in range(250)] + assert all_rows[0]["props"]["id"] == "node-0" + assert all_rows[0]["props"]["label"] == "Node 0" + + +def test_neo4j_nodes_are_grouped_by_sanitized_label(monkeypatch): + recorded: list = [] + monkeypatch.setitem(sys.modules, "neo4j", _fake_neo4j_module(recorded)) + from graphify.export import push_to_neo4j + + G = nx.MultiDiGraph() + G.add_node("a", file_type="python") + G.add_node("b", file_type="markdown") + G.add_node("c", file_type="python") + # Injection attempt must be sanitized, not parameterized away. + G.add_node("d", file_type="x) DETACH DELETE n //") + + result = push_to_neo4j(G, uri="bolt://x", user="neo4j", password="pw") + + assert result["nodes"] == 4 + labels = sorted(c.split("MERGE (n:")[1].split(" ")[0] for c, _ in recorded) + assert labels == ["Markdown", "Python", "Xdetachdeleten"] + by_label = {c.split("MERGE (n:")[1].split(" ")[0]: p["rows"] for c, p in recorded} + assert [r["id"] for r in by_label["Python"]] == ["a", "c"] + + +def test_neo4j_edges_are_grouped_and_batched(monkeypatch): + recorded: list = [] + monkeypatch.setitem(sys.modules, "neo4j", _fake_neo4j_module(recorded)) + from graphify.export import push_to_neo4j + + G = _make_graph(3) + G.add_edge("node-0", "node-1", relation="calls", confidence="INFERRED") + G.add_edge("node-1", "node-2", relation="calls") + G.add_edge("node-0", "node-2", relation="imports") + + result = push_to_neo4j(G, uri="bolt://x", user="neo4j", password="pw", + batch_size=100) + + assert result == {"nodes": 3, "edges": 3} + edge_queries = [(c, p) for c, p in recorded if "MATCH (a {id: row.src})" in c] + assert len(edge_queries) == 2 # one per relation: CALLS, IMPORTS + by_rel = {c.split("MERGE (a)-[r:")[1].split("]")[0]: p["rows"] + for c, p in edge_queries} + assert sorted(by_rel) == ["CALLS", "IMPORTS"] + assert [(r["src"], r["tgt"]) for r in by_rel["CALLS"]] == [ + ("node-0", "node-1"), ("node-1", "node-2") + ] + assert by_rel["CALLS"][0]["props"] == {"relation": "calls", "confidence": "INFERRED"} + + +def test_neo4j_community_lands_in_props(monkeypatch): + recorded: list = [] + monkeypatch.setitem(sys.modules, "neo4j", _fake_neo4j_module(recorded)) + from graphify.export import push_to_neo4j + + G = _make_graph(2) + push_to_neo4j(G, uri="bolt://x", user="neo4j", password="pw", + communities={7: ["node-1"]}) + + rows = recorded[0][1]["rows"] + by_id = {r["id"]: r["props"] for r in rows} + assert by_id["node-1"]["community"] == 7 + assert "community" not in by_id["node-0"] + + +def test_neo4j_rejects_nonpositive_batch_size(monkeypatch): + monkeypatch.setitem(sys.modules, "neo4j", _fake_neo4j_module([])) + from graphify.export import push_to_neo4j + + with pytest.raises(ValueError, match="batch_size"): + push_to_neo4j(_make_graph(1), uri="bolt://x", user="neo4j", + password="pw", batch_size=0) + + +# --------------------------------------------------------------------------- +# FalkorDB +# --------------------------------------------------------------------------- + +def test_falkordb_nodes_and_edges_are_batched(monkeypatch): + recorded: list = [] + monkeypatch.setitem(sys.modules, "falkordb", _fake_falkordb_module(recorded)) + from graphify.export import push_to_falkordb + + G = _make_graph(150) + G.add_edge("node-0", "node-1", relation="calls") + + result = push_to_falkordb(G, uri="localhost:6379", batch_size=100) + + assert result == {"nodes": 150, "edges": 1} + node_queries = [(c, p) for c, p in recorded if "MERGE (n:" in c] + edge_queries = [(c, p) for c, p in recorded if "MATCH (a {id: row.src})" in c] + assert len(node_queries) == 2 # 100 + 50 + assert [len(p["rows"]) for _, p in node_queries] == [100, 50] + assert len(edge_queries) == 1 + for cypher, params in recorded: + assert "UNWIND $rows AS row" in cypher + assert set(params) == {"rows"} # positional params dict, rows only + assert edge_queries[0][1]["rows"] == [ + {"src": "node-0", "tgt": "node-1", "props": {"relation": "calls"}} + ] + + +def test_falkordb_rejects_nonpositive_batch_size(monkeypatch): + monkeypatch.setitem(sys.modules, "falkordb", _fake_falkordb_module([])) + from graphify.export import push_to_falkordb + + with pytest.raises(ValueError, match="batch_size"): + push_to_falkordb(_make_graph(1), uri="localhost:6379", batch_size=-1) + + +# --------------------------------------------------------------------------- +# CLI flag validation (no graph, no driver, no network - rejected on parse) +# --------------------------------------------------------------------------- + +def _run_cli(args: list[str], cwd: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [PYTHON, "-m", "graphify"] + args, + cwd=cwd, + capture_output=True, + text=True, + ) + + +def test_cli_rejects_zero_batch_size(tmp_path): + proc = _run_cli(["export", "neo4j", "--push", "bolt://x", "--batch-size", "0"], + cwd=tmp_path) + assert proc.returncode == 2 + assert "error: --batch-size must be a positive integer" in proc.stderr + + +def test_cli_rejects_non_integer_batch_size(tmp_path): + proc = _run_cli(["export", "falkordb", "--push", "localhost:6379", + "--batch-size", "many"], cwd=tmp_path) + assert proc.returncode == 2 + assert "error: --batch-size must be an integer" in proc.stderr diff --git a/tests/test_extract_lock.py b/tests/test_extract_lock.py new file mode 100644 index 000000000..63624e05c --- /dev/null +++ b/tests/test_extract_lock.py @@ -0,0 +1,142 @@ +"""Tests for extract holding the per-repo rebuild lock. + +`graphify extract` takes the same advisory flock that ``_rebuild_code`` takes +so two extracts (or an extract and a watcher/hook rebuild) racing on one +graphify-out/ cannot interleave cache saves or clobber graph.json. These +tests cover the new bounded-wait mode of ``_rebuild_lock`` plus the CLI +behaviour under contention (run as a subprocess, like the other CLI tests). +""" +from __future__ import annotations +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +import pytest + +from graphify.watch import _rebuild_lock + +PYTHON = sys.executable +REPO_ROOT = Path(__file__).parent.parent + + +def _run_extract(args: list[str], cwd: Path, extra_env: dict[str, str] | None = None) -> subprocess.CompletedProcess: + env = dict(os.environ) + # Import THIS checkout in the subprocess, not an installed graphify. + env["PYTHONPATH"] = str(REPO_ROOT) + os.pathsep + env.get("PYTHONPATH", "") + if extra_env: + env.update(extra_env) + return subprocess.run( + [PYTHON, "-m", "graphify", "extract"] + args, + cwd=cwd, + capture_output=True, + text=True, + env=env, + ) + + +# --- _rebuild_lock bounded wait --- + + +@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") +def test_rebuild_lock_timeout_expires_yields_false(tmp_path): + """A blocking caller with a timeout must give up once the deadline passes + instead of waiting on the kernel forever, and must not disturb the + holder's lock file on the way out.""" + out = tmp_path / "graphify-out" + with _rebuild_lock(out) as outer: + assert outer is True + held_contents = (out / ".rebuild.lock").read_text(encoding="utf-8") + t0 = time.monotonic() + with _rebuild_lock(out, blocking=True, timeout=0.5) as inner: + assert inner is False + waited = time.monotonic() - t0 + assert waited >= 0.5, waited + assert waited < 5.0, waited + # The holder's PID payload must survive the failed bounded wait. + assert (out / ".rebuild.lock").read_text(encoding="utf-8") == held_contents + + +@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") +def test_rebuild_lock_timeout_acquires_when_free(tmp_path): + """The bounded-wait path must keep the PID-payload and unlink-on-release + contracts of the plain acquisition paths.""" + out = tmp_path / "graphify-out" + lock_path = out / ".rebuild.lock" + with _rebuild_lock(out, blocking=True, timeout=5.0) as got: + assert got is True + assert lock_path.read_text(encoding="utf-8") == f"{os.getpid()}\n" + assert not lock_path.exists(), "lock file should be unlinked after release" + + +@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") +def test_rebuild_lock_timeout_waits_for_release(tmp_path): + """A bounded waiter must acquire promptly once the holder releases, not + only when the deadline expires.""" + out = tmp_path / "graphify-out" + held = threading.Event() + release = threading.Event() + + def _holder(): + with _rebuild_lock(out) as got: + assert got is True + held.set() + release.wait(timeout=10) + + t = threading.Thread(target=_holder) + t.start() + try: + assert held.wait(timeout=10) + threading.Timer(0.5, release.set).start() + t0 = time.monotonic() + with _rebuild_lock(out, blocking=True, timeout=30.0) as got: + assert got is True + assert time.monotonic() - t0 < 10.0 + finally: + release.set() + t.join(timeout=10) + + +# --- graphify extract under contention (CLI, subprocess) --- + + +@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") +def test_extract_times_out_when_lock_is_held(tmp_path): + """extract must wait behind a held rebuild lock, name the holder's PID, + and exit 1 once GRAPHIFY_LOCK_TIMEOUT expires — not race the holder.""" + fcntl = pytest.importorskip("fcntl") + (tmp_path / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") + out = tmp_path / "graphify-out" + out.mkdir() + lock_path = out / ".rebuild.lock" + with open(lock_path, "a+", encoding="utf-8") as fh: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + fh.seek(0) + fh.truncate() + fh.write("12345\n") + fh.flush() + proc = _run_extract( + [str(tmp_path), "--code-only", "--no-cluster"], + cwd=tmp_path, + extra_env={"GRAPHIFY_LOCK_TIMEOUT": "1"}, + ) + assert proc.returncode == 1, proc.stdout + proc.stderr + assert "waiting for another rebuild (pid 12345)" in proc.stdout + assert "gave up waiting for the rebuild lock" in proc.stderr + # A timed-out waiter must not have clobbered the holder's payload. + assert lock_path.read_text(encoding="utf-8") == "12345\n" + + +@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") +def test_extract_releases_lock_when_free(tmp_path): + """An uncontended extract must run to completion and leave no lock file + behind (the unlink-on-release contract downstream pollers rely on).""" + (tmp_path / "app.py").write_text( + "def f():\n return 1\n\ndef g():\n return f()\n", encoding="utf-8" + ) + proc = _run_extract([str(tmp_path), "--code-only", "--no-cluster"], cwd=tmp_path) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert (tmp_path / "graphify-out" / "graph.json").exists() + assert not (tmp_path / "graphify-out" / ".rebuild.lock").exists() diff --git a/tests/test_fallback_backend.py b/tests/test_fallback_backend.py new file mode 100644 index 000000000..fa9cf91af --- /dev/null +++ b/tests/test_fallback_backend.py @@ -0,0 +1,267 @@ +"""Tests for `graphify extract --fallback-backend` (retry on a second brain). + +When EVERY semantic chunk fails on the primary backend (missing SDK, bad key, +an outage), a configured fallback backend retries the same still-uncached +files once — nothing was cache-saved for a zero-success pass, so the retry +covers exactly the files the primary failed on. Only when the fallback also +ends at zero successes does extract keep the all-chunks-failed exit 1. +Without a fallback configured, behavior is unchanged. +""" +from __future__ import annotations + +import pytest + +import graphify.__main__ as mainmod + + +def _make_corpus(tmp_path): + """Minimal corpus: one Go code file + one Markdown doc. + + Both file types are needed so semantic extraction is requested + (docs path triggers the LLM step the fallback wraps). + """ + (tmp_path / "main.go").write_text("package main\nfunc main() {}\n") + (tmp_path / "README.md").write_text("# Notes\nThe main function entry point.\n") + return tmp_path + + +def _recording_stub(calls, *, fail_backends=(), raise_backends=None): + """Stub extract_corpus_parallel that records each dispatch. + + Backends in ``fail_backends`` simulate "all chunks failed": an empty + accumulator, on_chunk_done never invoked. Backends in ``raise_backends`` + (a dict backend -> exception) crash the whole pass. Everything else + succeeds with one chunk. + """ + raise_backends = raise_backends or {} + + def _stub(paths, **kwargs): + be = kwargs.get("backend") + calls.append({ + "backend": be, + "model": kwargs.get("model"), + "paths": sorted(str(p) for p in paths), + }) + if be in raise_backends: + raise raise_backends[be] + if be in fail_backends: + return {"nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0} + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + return {"nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 10, "output_tokens": 5} + + return _stub + + +def _arm(monkeypatch, tmp_path, stub, *, extra_argv=(), env=None): + corpus = _make_corpus(tmp_path) + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FALLBACK_BACKEND", raising=False) + for key, value in (env or {}).items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", stub) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--out", str(out_dir), *extra_argv], + ) + return corpus, out_dir + + +def _run_ok(capsys=None): + # extract may still raise SystemExit at the end (clean exit code 0) + # depending on platform; accept either no exception or SystemExit(0). + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + +def test_fallback_retries_same_paths_and_succeeds(monkeypatch, tmp_path, capsys): + calls = [] + stub = _recording_stub(calls, fail_backends=("claude",)) + corpus, out_dir = _arm( + monkeypatch, tmp_path, stub, + extra_argv=["--fallback-backend", "openai", "--model", "claude-test-model"], + ) + + _run_ok() + + assert [c["backend"] for c in calls] == ["claude", "openai"] + # The retry covers exactly the files the primary failed on. + assert calls[0]["paths"] == calls[1]["paths"] == [str(corpus / "README.md")] + # --model names a model on the PRIMARY backend only; the fallback runs + # on its own default model. + assert calls[0]["model"] == "claude-test-model" + assert calls[1]["model"] is None + out = capsys.readouterr().out + assert "retrying once with fallback backend 'openai'" in out + assert (out_dir / "graphify-out" / "graph.json").exists(), ( + "graph.json must be written when the fallback pass succeeds" + ) + + +def test_fallback_also_failing_keeps_exit_1(monkeypatch, tmp_path, capsys): + calls = [] + stub = _recording_stub(calls, fail_backends=("claude", "openai")) + _corpus, out_dir = _arm( + monkeypatch, tmp_path, stub, extra_argv=["--fallback-backend", "openai"], + ) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert [c["backend"] for c in calls] == ["claude", "openai"], ( + "the fallback must be tried exactly once before failing" + ) + err = capsys.readouterr().err + assert "all semantic chunks failed" in err + assert "openai" in err, "the final error must name the last backend tried" + assert not (out_dir / "graphify-out" / "graph.json").exists() + + +def test_no_fallback_behavior_is_unchanged(monkeypatch, tmp_path, capsys): + calls = [] + stub = _recording_stub(calls, fail_backends=("claude",)) + _arm(monkeypatch, tmp_path, stub) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert [c["backend"] for c in calls] == ["claude"] + err = capsys.readouterr().err + assert "all semantic chunks failed" in err + assert "claude" in err + + +def test_fallback_equal_to_primary_is_not_retried(monkeypatch, tmp_path): + # Retrying the very backend that just zeroed out would double the spend + # for the same outcome, so an identical fallback is a no-op. + calls = [] + stub = _recording_stub(calls, fail_backends=("claude",)) + _arm(monkeypatch, tmp_path, stub, extra_argv=["--fallback-backend", "claude"]) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert [c["backend"] for c in calls] == ["claude"] + + +def test_unknown_fallback_backend_is_rejected_upfront(monkeypatch, tmp_path, capsys): + calls = [] + stub = _recording_stub(calls) + _arm(monkeypatch, tmp_path, stub, extra_argv=["--fallback-backend", "warpdrive"]) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + err = capsys.readouterr().err + assert "unknown fallback backend 'warpdrive'" in err + assert calls == [], "a typo'd fallback must fail before any API dispatch" + + +def test_env_var_arms_the_fallback(monkeypatch, tmp_path, capsys): + calls = [] + stub = _recording_stub(calls, fail_backends=("claude",)) + _arm(monkeypatch, tmp_path, stub, env={"GRAPHIFY_FALLBACK_BACKEND": "openai"}) + + _run_ok() + + assert [c["backend"] for c in calls] == ["claude", "openai"] + + +def test_cli_flag_wins_over_env_var(monkeypatch, tmp_path): + calls = [] + stub = _recording_stub(calls, fail_backends=("claude",)) + _arm( + monkeypatch, tmp_path, stub, + extra_argv=["--fallback-backend=kimi"], + env={"GRAPHIFY_FALLBACK_BACKEND": "openai"}, + ) + + _run_ok() + + assert [c["backend"] for c in calls] == ["claude", "kimi"] + + +def test_primary_crash_triggers_fallback(monkeypatch, tmp_path, capsys): + # A pass that raises leaves zero succeeded chunks, so it rides the same + # retry path as per-chunk total failure. + calls = [] + stub = _recording_stub( + calls, raise_backends={"claude": RuntimeError("backend melted")}, + ) + _arm(monkeypatch, tmp_path, stub, extra_argv=["--fallback-backend", "openai"]) + + _run_ok() + + assert [c["backend"] for c in calls] == ["claude", "openai"] + captured = capsys.readouterr() + assert "semantic extraction failed: backend melted" in captured.err + assert "retrying once with fallback backend 'openai'" in captured.out + + +def test_primary_import_error_falls_back_instead_of_dying(monkeypatch, tmp_path, capsys): + # A missing SDK package is fatal without a fallback, but with one + # configured it is exactly the case the fallback exists for. + calls = [] + stub = _recording_stub( + calls, raise_backends={"claude": ImportError("requires the anthropic package")}, + ) + _arm(monkeypatch, tmp_path, stub, extra_argv=["--fallback-backend", "openai"]) + + _run_ok() + + assert [c["backend"] for c in calls] == ["claude", "openai"] + assert "requires the anthropic package" in capsys.readouterr().err + + +def test_partial_primary_success_does_not_fire_fallback(monkeypatch, tmp_path, capsys): + # The fallback exists for TOTAL failure only. When the primary got at + # least one chunk through, retrying the whole set on a second backend + # would re-spend on the chunks that already succeeded — so a partial + # pass keeps its result (and the incomplete-build guard), no retry. + calls = [] + + def _partial_stub(paths, **kwargs): + calls.append({"backend": kwargs.get("backend")}) + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + # 1 of 2 chunks succeeded; the second never reports. + on_chunk(0, 2, {"nodes": [], "edges": [], "hyperedges": []}) + return {"nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 10, "output_tokens": 5} + + _arm(monkeypatch, tmp_path, _partial_stub, + extra_argv=["--fallback-backend", "openai"]) + + _run_ok() + + assert [c["backend"] for c in calls] == ["claude"], ( + "a partially-succeeded primary pass must not re-dispatch on the fallback" + ) + assert "retrying once with fallback backend" not in capsys.readouterr().out + + +def test_import_error_without_fallback_stays_fatal(monkeypatch, tmp_path, capsys): + calls = [] + stub = _recording_stub( + calls, raise_backends={"claude": ImportError("requires the anthropic package")}, + ) + _arm(monkeypatch, tmp_path, stub) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert [c["backend"] for c in calls] == ["claude"] + assert "requires the anthropic package" in capsys.readouterr().err diff --git a/tests/test_kimi_reasoning_effort.py b/tests/test_kimi_reasoning_effort.py new file mode 100644 index 000000000..f4cab00f3 --- /dev/null +++ b/tests/test_kimi_reasoning_effort.py @@ -0,0 +1,31 @@ +"""Tests for the kimi backend's reasoning_effort config (GRAPHIFY_KIMI_EFFORT). + +Kimi K3 advertises valid_efforts ["low","high","max"] on /models; sending +nothing let the server default ("high") apply silently. The backend config now +carries an explicit effort, defaulting to "max", overridable via env — the +same import-time pattern as ANTHROPIC_BASE_URL on the claude backend. +""" + +import importlib + +from graphify import llm + + +def test_kimi_reasoning_effort_defaults_to_max(monkeypatch): + monkeypatch.delenv("GRAPHIFY_KIMI_EFFORT", raising=False) + reloaded = importlib.reload(llm) + try: + assert reloaded.BACKENDS["kimi"]["reasoning_effort"] == "max" + finally: + monkeypatch.undo() + importlib.reload(llm) + + +def test_kimi_reasoning_effort_env_override(monkeypatch): + monkeypatch.setenv("GRAPHIFY_KIMI_EFFORT", "low") + reloaded = importlib.reload(llm) + try: + assert reloaded.BACKENDS["kimi"]["reasoning_effort"] == "low" + finally: + monkeypatch.undo() + importlib.reload(llm) diff --git a/tests/test_openai_cli_backend.py b/tests/test_openai_cli_backend.py new file mode 100644 index 000000000..193b9a955 --- /dev/null +++ b/tests/test_openai_cli_backend.py @@ -0,0 +1,219 @@ +"""Tests for the openai-cli backend (_call_openai_cli). + +Covers the argv contract, the happy JSON parse, the failure paths (non-zero +exit, missing/empty -o output, hollow graph), and turn-usage accounting. +No network, no Codex binary: shutil.which and subprocess.run are monkeypatched. +""" +import json + +import pytest + +from graphify import llm + + +class _Captured: + def __init__(self): + self.args = None + self.kwargs = None + self.calls = [] + + +def _arm(monkeypatch, response=None, servers=("graphify", "docs"), + exec_returncode=0, exec_stdout="", exec_stderr="", write_output=True): + """Fake a Codex CLI: `mcp list --json` returns servers, `exec` writes the -o file.""" + cap = _Captured() + import shutil + monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/codex" if name == "codex" else None) + + class P: + def __init__(self, stdout="", returncode=0, stderr=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def fake_run(args, **kwargs): + cap.calls.append(list(args)) + if "mcp" in args and "list" in args: + return P(json.dumps([{"name": n, "enabled": True} for n in servers])) + cap.args = list(args) + cap.kwargs = kwargs + # write the -o file the way codex exec does + out_idx = args.index("-o") + 1 + if write_output: + payload = response if response is not None else {"nodes": [{"id": "f", "type": "function"}], "edges": []} + if isinstance(payload, str): + with open(args[out_idx], "w", encoding="utf-8") as fh: + fh.write(payload) + else: + with open(args[out_idx], "w", encoding="utf-8") as fh: + json.dump(payload, fh) + else: + # simulate codex never producing the file (the backend pre-creates + # the temp path, so "missing" means it is gone by the time we look) + import os as _os + try: + _os.unlink(args[out_idx]) + except OSError: + pass + return P(stdout=exec_stdout, returncode=exec_returncode, stderr=exec_stderr) + + import subprocess as _sp + monkeypatch.setattr(_sp, "run", fake_run) + return cap + + +def test_argv_contract_defaults(monkeypatch): + monkeypatch.delenv("GRAPHIFY_OPENAI_CLI_MODEL", raising=False) + monkeypatch.delenv("GRAPHIFY_OPENAI_CLI_EFFORT", raising=False) + cap = _arm(monkeypatch) + llm._call_openai_cli("def f(): pass", max_tokens=64) + a = cap.args + assert a[0] == "/usr/bin/codex" and a[1] == "exec" + assert "--skip-git-repo-check" in a and "--json" in a + assert a[a.index("--sandbox") + 1] == "read-only" + assert a[a.index("--model") + 1] == "gpt-5.6-sol" # default model + assert "model_reasoning_effort=ultra" in a # default effort + assert a[-1] == "-" # prompt via stdin + assert cap.kwargs.get("input") # not argv (MAX_ARG_STRLEN) + + +def test_every_configured_mcp_server_is_disabled_per_call(monkeypatch): + """A blanket `mcp_servers={}` is merged away by Codex; per-server `enabled` works.""" + cap = _arm(monkeypatch, servers=("graphify", "docs")) + llm._call_openai_cli("def f(): pass", max_tokens=64) + a = cap.args + assert "mcp_servers.graphify.enabled=false" in a + assert "mcp_servers.docs.enabled=false" in a + assert "mcp_servers={}" not in a + # the server list came from Codex itself, no hardcoded names + assert any("mcp" in c and "list" in c and "--json" in c for c in cap.calls) + + +def test_no_configured_servers_adds_no_overrides(monkeypatch): + cap = _arm(monkeypatch, servers=()) + llm._call_openai_cli("def f(): pass", max_tokens=64) + assert not [x for x in cap.args if str(x).startswith("mcp_servers.")] + + +def test_argv_env_overrides(monkeypatch): + monkeypatch.setenv("GRAPHIFY_OPENAI_CLI_MODEL", "gpt-5.6-luna") + monkeypatch.setenv("GRAPHIFY_OPENAI_CLI_EFFORT", "high") + cap = _arm(monkeypatch) + llm._call_openai_cli("def f(): pass", max_tokens=64) + a = cap.args + assert a[a.index("--model") + 1] == "gpt-5.6-luna" + assert "model_reasoning_effort=high" in a + assert "model_reasoning_effort=ultra" not in a + + +def test_missing_binary_raises(monkeypatch): + import shutil + monkeypatch.setattr(shutil, "which", lambda name: None) + with pytest.raises(RuntimeError, match="Codex CLI not found"): + llm._call_openai_cli("x", max_tokens=16) + + +# --------------------------------------------------------------------------- +# Happy path: the -o JSON becomes the result dict, usage rides the JSONL stdout +# --------------------------------------------------------------------------- + +def test_happy_parse_returns_graph_and_usage(monkeypatch): + monkeypatch.delenv("GRAPHIFY_OPENAI_CLI_MODEL", raising=False) + stdout = "\n".join([ + json.dumps({"type": "turn.started"}), + "not-json diagnostics line", + json.dumps({"type": "turn.completed", + "usage": {"input_tokens": 1200, "cached_input_tokens": 1000, + "output_tokens": 345}}), + ]) + payload = {"nodes": [{"id": "acme.f", "type": "function"}], + "edges": [{"source": "acme.f", "target": "acme.g", "relation": "calls"}]} + _arm(monkeypatch, response=payload, exec_stdout=stdout) + result = llm._call_openai_cli("def f(): g()", max_tokens=64) + assert result["nodes"] == payload["nodes"] + assert result["edges"] == payload["edges"] + # input_tokens already includes cached_input_tokens; must not be re-added + assert result["input_tokens"] == 1200 + assert result["output_tokens"] == 345 + assert result["model"] == "gpt-5.6-sol" + assert result["finish_reason"] == "stop" + + +# --------------------------------------------------------------------------- +# Failure paths: every raise carries the vendor detail (stderr + stdout tail) +# --------------------------------------------------------------------------- + +def test_nonzero_exit_raises_with_vendor_detail(monkeypatch): + _arm(monkeypatch, exec_returncode=2, + exec_stderr="ERROR: 400 Bad Request: model not found", + exec_stdout=json.dumps({"type": "error", "message": "http 400"})) + with pytest.raises(RuntimeError) as exc: + llm._call_openai_cli("x", max_tokens=16) + msg = str(exc.value) + assert "codex exec exited 2" in msg + assert "400 Bad Request" in msg # stderr preserved + assert "http 400" in msg # JSONL stdout tail preserved + + +def test_missing_output_file_raises(monkeypatch): + _arm(monkeypatch, write_output=False, exec_stderr="boom") + with pytest.raises(RuntimeError, match="produced no -o output file"): + llm._call_openai_cli("x", max_tokens=16) + + +def test_empty_output_file_raises(monkeypatch): + _arm(monkeypatch, response=" \n", exec_stderr="quota hint on stderr") + with pytest.raises(RuntimeError) as exc: + llm._call_openai_cli("x", max_tokens=16) + msg = str(exc.value) + assert "empty -o output file" in msg + assert "quota hint on stderr" in msg + + +def test_empty_graph_raises_instead_of_hollow_bisect(monkeypatch): + # An empty-but-valid graph must raise, or _response_is_hollow would bisect + # the chunk into up to 15 more subscription calls. + _arm(monkeypatch, response={"nodes": [], "edges": []}) + with pytest.raises(RuntimeError, match="returned no graph content"): + llm._call_openai_cli("x", max_tokens=16) + + +# --------------------------------------------------------------------------- +# _openai_cli_turn_usage: JSONL accounting +# --------------------------------------------------------------------------- + +def test_turn_usage_reads_last_completed_event(): + stdout = "\n".join([ + json.dumps({"type": "turn.completed", "usage": {"input_tokens": 1, "output_tokens": 2}}), + json.dumps({"type": "turn.completed", "usage": {"input_tokens": 30, "output_tokens": 40}}), + ]) + assert llm._openai_cli_turn_usage(stdout) == (30, 40) + + +def test_turn_usage_tolerates_malformed_lines(): + stdout = "\n".join([ + "plain diagnostics", + "{not json", + json.dumps({"type": "turn.completed", "usage": {"input_tokens": 7, "output_tokens": 8}}), + ]) + assert llm._openai_cli_turn_usage(stdout) == (7, 8) + + +def test_turn_usage_without_completed_event_reports_zero(): + assert llm._openai_cli_turn_usage("") == (0, 0) + assert llm._openai_cli_turn_usage(json.dumps({"type": "turn.started"})) == (0, 0) + + +def test_turn_usage_non_numeric_counts_are_zero(): + stdout = json.dumps({"type": "turn.completed", + "usage": {"input_tokens": "many", "output_tokens": None}}) + assert llm._openai_cli_turn_usage(stdout) == (0, 0) + + +def test_vendor_detail_bounds_and_labels(): + detail = llm._openai_cli_vendor_detail("E" * 1000, "O" * 1000) + assert detail.startswith("stderr: ") + assert "stdout tail: " in detail + # both sides bounded to their last 400 chars + assert len(detail) < 900 + assert llm._openai_cli_vendor_detail("", "") == "(no stderr or stdout)" diff --git a/tests/test_watch_semantic.py b/tests/test_watch_semantic.py new file mode 100644 index 000000000..e44576b03 --- /dev/null +++ b/tests/test_watch_semantic.py @@ -0,0 +1,144 @@ +"""Tests for `graphify watch --semantic` - automatic LLM extraction on doc changes. + +The extract subprocess is always mocked: no test here may reach a real +backend. CLI arg-validation tests drive `python -m graphify watch` as a +subprocess but only down paths that exit before the watcher starts. +""" +from __future__ import annotations +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from graphify.watch import _run_semantic_extract, _GRAPHIFY_OUT + +PYTHON = sys.executable +REPO_ROOT = Path(__file__).resolve().parent.parent + + +class _FakeCompleted: + def __init__(self, rc: int): + self.returncode = rc + + +def _patch_run(monkeypatch, rc: int, calls: list): + def fake_run(cmd, *args, **kwargs): + calls.append(cmd) + return _FakeCompleted(rc) + monkeypatch.setattr(subprocess, "run", fake_run) + + +# --- _run_semantic_extract: subprocess command shape --- + +def test_semantic_extract_invokes_module_cli(tmp_path, monkeypatch): + calls: list = [] + _patch_run(monkeypatch, 0, calls) + assert _run_semantic_extract(tmp_path) is True + assert calls == [[PYTHON, "-m", "graphify", "extract", str(tmp_path)]] + +def test_semantic_extract_forwards_backend_flags(tmp_path, monkeypatch): + calls: list = [] + _patch_run(monkeypatch, 0, calls) + _run_semantic_extract(tmp_path, backend="gemini", fallback_backend="ollama") + assert calls[0] == [ + PYTHON, "-m", "graphify", "extract", str(tmp_path), + "--backend", "gemini", "--fallback-backend", "ollama", + ] + +def test_semantic_extract_omits_unset_backend_flags(tmp_path, monkeypatch): + # An unset backend must not appear as `--backend None` in the child argv. + calls: list = [] + _patch_run(monkeypatch, 0, calls) + _run_semantic_extract(tmp_path) + assert "--backend" not in calls[0] + assert "--fallback-backend" not in calls[0] + + +# --- _run_semantic_extract: needs_update flag contract --- + +def test_semantic_extract_clears_stale_flag_on_success(tmp_path, monkeypatch): + """Extract never touches the flag itself (only _rebuild_code does), so a + successful semantic run must clear it here or the user is left with a + stale 'run /graphify --update' prompt.""" + flag = tmp_path / _GRAPHIFY_OUT / "needs_update" + flag.parent.mkdir(parents=True) + flag.write_text("1", encoding="utf-8") + _patch_run(monkeypatch, 0, []) + assert _run_semantic_extract(tmp_path) is True + assert not flag.exists() + +def test_semantic_extract_failure_keeps_flag(tmp_path, monkeypatch, capsys): + """A failed extract must leave the flag alone and return False so the + watcher falls back to _notify_only - never swallow the failure.""" + flag = tmp_path / _GRAPHIFY_OUT / "needs_update" + flag.parent.mkdir(parents=True) + flag.write_text("1", encoding="utf-8") + _patch_run(monkeypatch, 1, []) + assert _run_semantic_extract(tmp_path) is False + assert flag.exists() + assert "exited with code 1" in capsys.readouterr().out + +def test_semantic_extract_success_without_flag_is_fine(tmp_path, monkeypatch): + # No pre-existing flag (the common case: the batch never wrote one). + _patch_run(monkeypatch, 0, []) + assert _run_semantic_extract(tmp_path) is True + +def test_semantic_extract_oserror_returns_false(tmp_path, monkeypatch, capsys): + def boom(cmd, *args, **kwargs): + raise OSError("no such interpreter") + monkeypatch.setattr(subprocess, "run", boom) + assert _run_semantic_extract(tmp_path) is False + assert "failed to start" in capsys.readouterr().out + + +# --- watch() signature: semantic knobs are keyword-only --- + +def test_watch_semantic_params_are_keyword_only(): + import inspect + from graphify.watch import watch + params = inspect.signature(watch).parameters + for name in ("semantic", "backend", "fallback_backend"): + assert params[name].kind is inspect.Parameter.KEYWORD_ONLY + + +# --- CLI arg validation (exits before the watcher starts; no watchdog needed) --- + +def _run_cli(args: list[str], cwd: Path) -> subprocess.CompletedProcess: + # Pin PYTHONPATH to this checkout so the subprocess exercises the code + # under test even when a different graphify is installed site-wide. + env = os.environ.copy() + env["PYTHONPATH"] = str(REPO_ROOT) + os.pathsep + env.get("PYTHONPATH", "") + return subprocess.run( + [PYTHON, "-m", "graphify", "watch"] + args, + cwd=cwd, capture_output=True, text=True, timeout=60, env=env, + ) + +def test_cli_watch_backend_without_semantic_rejected(tmp_path): + """Without --semantic no extract ever runs, so a backend flag would be a + silent no-op - it must be rejected loudly.""" + r = _run_cli(["--backend", "gemini"], tmp_path) + assert r.returncode == 2 + assert "--semantic" in r.stderr + +def test_cli_watch_fallback_backend_without_semantic_rejected(tmp_path): + r = _run_cli(["--fallback-backend=ollama"], tmp_path) + assert r.returncode == 2 + assert "--semantic" in r.stderr + +def test_cli_watch_unknown_option_rejected(tmp_path): + r = _run_cli(["--sematnic"], tmp_path) + assert r.returncode == 2 + assert "unknown watch option" in r.stderr + +def test_cli_watch_two_paths_rejected(tmp_path): + r = _run_cli([str(tmp_path), str(tmp_path)], tmp_path) + assert r.returncode == 2 + assert "at most one path" in r.stderr + +def test_cli_watch_semantic_missing_path_errors(tmp_path): + # Flag parsing must leave the positional path intact. + r = _run_cli(["--semantic", "no-such-dir"], tmp_path) + assert r.returncode == 1 + assert "path not found" in r.stderr