From 3e7cda43ee2bcc804e315dfaa76c790092cfd002 Mon Sep 17 00:00:00 2001 From: antonioscarinci <96891485+antonioscarinci@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:16:30 +0200 Subject: [PATCH 1/3] feat(windows): merge windows-skill-fix into v8 as windows-v8-merged Combines v8's features (uv/pipx detection, graphify-out/ convention, FalkorDB, Step 4.5 health check, directed-graph support, shrink-guard, build_merge --update) with windows-skill-fix's approach (PowerShell throughout, external windows-scripts/*.py, fail-fast, incremental --update fix, Windows encoding fixes). Key changes: - windows-scripts/: all 34 scripts ported to graphify-out/ paths and v8 API (root=, cache_root=, directed=, build_from_json, etc.) - detect_incremental.py: propagates all_files so manifest covers full corpus on --update, not just changed files - check_graph_health.py: new script wrapping v8's diagnostics step - skill-windows.md: rewritten for PowerShell with $GRAPHIFY_SCRIPTS loaded from file each step; --update uses inline build_merge() so edge direction is preserved (#801, #1344, #1361) - export.py: replace None attribute values before nx.write_graphml to avoid AttributeError on None-valued node/edge properties - __main__.py: add --answer-file to graphify save-result to bypass Windows command-line length limit for long answers - .gitattributes: add line-ending normalization rules for .py/.sh (LF) and .bat/.cmd (CRLF) Co-Authored-By: Claude Sonnet 4.6 --- .gitattributes | 16 + graphify/__main__.py | 7 +- graphify/export.py | 9 + graphify/skill-windows.md | 663 ++++++++---------- .../add_transcripts_to_detect.py | 23 + graphify/windows-scripts/ast_extraction.py | 20 + graphify/windows-scripts/build_graph.py | 47 ++ graphify/windows-scripts/check_code_only.py | 12 + .../windows-scripts/check_extraction_cache.py | 26 + .../windows-scripts/check_graph_exists.py | 8 + .../windows-scripts/check_graph_health.py | 23 + .../check_graphify_installed.py | 4 + graphify/windows-scripts/cluster_only.py | 37 + .../windows-scripts/collect_chunk_results.py | 58 ++ graphify/windows-scripts/detect_files.py | 20 + .../windows-scripts/detect_incremental.py | 39 ++ graphify/windows-scripts/explain_node.py | 37 + graphify/windows-scripts/export_graphml.py | 16 + graphify/windows-scripts/export_html.py | 21 + .../windows-scripts/export_neo4j_cypher.py | 11 + graphify/windows-scripts/export_neo4j_push.py | 19 + graphify/windows-scripts/export_obsidian.py | 28 + graphify/windows-scripts/export_svg.py | 22 + graphify/windows-scripts/export_wiki.py | 22 + graphify/windows-scripts/graph_diff.py | 22 + graphify/windows-scripts/ingest_url.py | 19 + graphify/windows-scripts/label_communities.py | 37 + .../windows-scripts/merge_ast_semantic.py | 39 ++ graphify/windows-scripts/merge_graphs.py | 49 ++ .../windows-scripts/merge_semantic_results.py | 28 + .../windows-scripts/path_between_nodes.py | 45 ++ .../windows-scripts/print_detect_summary.py | 54 ++ .../windows-scripts/print_timing_estimate.py | 20 + graphify/windows-scripts/query_graph.py | 79 +++ graphify/windows-scripts/run_benchmark.py | 14 + .../windows-scripts/save_manifest_and_cost.py | 38 + .../windows-scripts/save_results_to_cache.py | 11 + graphify/windows-scripts/transcribe_video.py | 13 + graphify/windows-scripts/write_python_path.py | 7 + 39 files changed, 1299 insertions(+), 364 deletions(-) create mode 100644 graphify/windows-scripts/add_transcripts_to_detect.py create mode 100644 graphify/windows-scripts/ast_extraction.py create mode 100644 graphify/windows-scripts/build_graph.py create mode 100644 graphify/windows-scripts/check_code_only.py create mode 100644 graphify/windows-scripts/check_extraction_cache.py create mode 100644 graphify/windows-scripts/check_graph_exists.py create mode 100644 graphify/windows-scripts/check_graph_health.py create mode 100644 graphify/windows-scripts/check_graphify_installed.py create mode 100644 graphify/windows-scripts/cluster_only.py create mode 100644 graphify/windows-scripts/collect_chunk_results.py create mode 100644 graphify/windows-scripts/detect_files.py create mode 100644 graphify/windows-scripts/detect_incremental.py create mode 100644 graphify/windows-scripts/explain_node.py create mode 100644 graphify/windows-scripts/export_graphml.py create mode 100644 graphify/windows-scripts/export_html.py create mode 100644 graphify/windows-scripts/export_neo4j_cypher.py create mode 100644 graphify/windows-scripts/export_neo4j_push.py create mode 100644 graphify/windows-scripts/export_obsidian.py create mode 100644 graphify/windows-scripts/export_svg.py create mode 100644 graphify/windows-scripts/export_wiki.py create mode 100644 graphify/windows-scripts/graph_diff.py create mode 100644 graphify/windows-scripts/ingest_url.py create mode 100644 graphify/windows-scripts/label_communities.py create mode 100644 graphify/windows-scripts/merge_ast_semantic.py create mode 100644 graphify/windows-scripts/merge_graphs.py create mode 100644 graphify/windows-scripts/merge_semantic_results.py create mode 100644 graphify/windows-scripts/path_between_nodes.py create mode 100644 graphify/windows-scripts/print_detect_summary.py create mode 100644 graphify/windows-scripts/print_timing_estimate.py create mode 100644 graphify/windows-scripts/query_graph.py create mode 100644 graphify/windows-scripts/run_benchmark.py create mode 100644 graphify/windows-scripts/save_manifest_and_cost.py create mode 100644 graphify/windows-scripts/save_results_to_cache.py create mode 100644 graphify/windows-scripts/transcribe_video.py create mode 100644 graphify/windows-scripts/write_python_path.py diff --git a/.gitattributes b/.gitattributes index faa0f4b26..8c883a87e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,19 @@ worked/**/*.html linguist-vendored=true graphify-out/**/*.html linguist-vendored=true *.html linguist-detectable=false + +# Normalize line endings +* text=auto + +# Scripts always get LF +*.sh text eol=lf +*.py text eol=lf + +# Windows batch files always get CRLF +*.bat text eol=crlf +*.cmd text eol=crlf + +# Binary files +*.pdf binary +*.png binary +*.jpg binary diff --git a/graphify/__main__.py b/graphify/__main__.py index 157a48a61..413b02c02 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2932,13 +2932,18 @@ def main() -> None: p = _ap.ArgumentParser(prog="graphify save-result") p.add_argument("--question", required=True) - p.add_argument("--answer", required=True) + p.add_argument("--answer", default=None) + p.add_argument("--answer-file", dest="answer_file", default=None) p.add_argument("--type", dest="query_type", default="query") p.add_argument("--nodes", nargs="*", default=[]) p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) p.add_argument("--correction", default=None) p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) opts = p.parse_args(sys.argv[2:]) + if opts.answer_file: + opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() + elif not opts.answer: + p.error("--answer or --answer-file is required") from graphify.ingest import save_query_result as _sqr out = _sqr( diff --git a/graphify/export.py b/graphify/export.py index 7e98f47fc..ce07b69b6 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1490,6 +1490,15 @@ def to_graphml( for _, _, attrs in H.edges(data=True): for k in [k for k in attrs if k.startswith("_")]: del attrs[k] + # nx.write_graphml raises ValueError on None attribute values; replace with "". + for node_id in H.nodes(): + for key, val in list(H.nodes[node_id].items()): + if val is None: + H.nodes[node_id][key] = "" + for u, v in H.edges(): + for key, val in list(H.edges[u, v].items()): + if val is None: + H.edges[u, v][key] = "" nx.write_graphml(H, output_path) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index f6c83fb81..5d9f38fb3 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -116,56 +116,69 @@ if (-not $GRAPHIFY_PYTHON) { $GRAPHIFY_PYTHON = Find-GraphifyPython } -# Save interpreter path — all subsequent steps read this +# Fail fast — every subsequent step needs the interpreter +if (-not $GRAPHIFY_PYTHON) { + Write-Error '[graphify] ERROR: Python with graphify not found. Install with: uv tool install graphifyy (or pip install graphifyy)' + exit 1 +} + +# Save interpreter path and scripts path — all subsequent steps read these from disk $GRAPHIFY_PYTHON | Out-File -FilePath graphify-out\.graphify_python -Encoding utf8 -NoNewline -# Save scan root so `graphify update` (no args) knows where to look next time +$GRAPHIFY_SCRIPTS = (& $GRAPHIFY_PYTHON -c "import graphify, os; print(os.path.join(os.path.dirname(graphify.__file__), 'windows-scripts'))").Trim() +$GRAPHIFY_SCRIPTS | Out-File -FilePath graphify-out\.graphify_scripts -Encoding utf8 -NoNewline +# Save scan root so `--update` (no args) knows where to look next time (Resolve-Path INPUT_PATH).Path | Out-File -FilePath graphify-out\.graphify_root -Encoding utf8 -NoNewline ``` -If the import succeeds, print nothing and move straight to Step 2. +If Step 1 succeeds, print nothing and move straight to Step 2. -**In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** +**In every subsequent block**, load the interpreter and scripts paths from disk at the top — each PowerShell tool call is a fresh process and cannot inherit variables from previous calls: +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +``` ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\detect_files.py" "INPUT_PATH" +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\print_detect_summary.py" +$exitCode = $LASTEXITCODE ``` -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Replace INPUT_PATH with the actual path the user provided. Relay the `print_detect_summary.py` output verbatim to the user, then: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` +- If `$exitCode` is 1: stop — no supported files found. +- If `$exitCode` is 2: the output already lists the top 5 subdirectories; ask the user which subfolder to run on and wait for their answer before proceeding. +- If the output contains `HAS_VIDEO=true`: proceed to Step 2.5; otherwise skip to Step 3. -Omit any category with 0 files from the summary. +### Step 2.5 - Video and audio (only if video files detected) -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Skip this step entirely if `detect` returned zero `video` files. -### Step 2.5 - Video and audio (only if video files detected) +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Step 1 - Write the Whisper prompt.** Read the top node labels from the detection output, then compose a short domain hint sentence. For example: +- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +If the corpus has only video files with no other docs/code, use: `"Use proper punctuation and paragraph breaks."` -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +Set it as `$env:GRAPHIFY_WHISPER_PROMPT` before running transcription. + +**Step 2 - Transcribe:** + +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +# If --whisper-model was passed: $env:GRAPHIFY_WHISPER_MODEL = "" +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\transcribe_video.py" | Out-File -FilePath graphify-out\.graphify_transcripts.json -Encoding utf8 +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\add_transcripts_to_detect.py" +``` + +`add_transcripts_to_detect.py` merges transcript paths into `graphify-out/.graphify_detect.json` so they are treated as docs in Step 3. Relay its output to the user. ### Step 3 - Extract entities and relationships @@ -184,81 +197,43 @@ Print it once, then continue — do not wait for the user to supply a key. If `G **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. - #### Part A - Structural extraction for code files For any code files detected, run AST extraction in parallel with Part B subagents: -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\ast_extraction.py" "INPUT_PATH" ``` #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code — there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +& $GRAPHIFY_PYTHON -c "import json; from pathlib import Path; Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8')" ``` **MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" + +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\print_timing_estimate.py" +``` **Step B0 - Check extraction cache first** Before dispatching any subagents, check which files already have cached extraction results: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\check_extraction_cache.py" "INPUT_PATH" ``` Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. @@ -295,318 +270,160 @@ See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, **Step B3 - Collect, cache, and merge** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort +Wait for all subagents. After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging — the chunk JSON itself always has placeholder zeros. -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path +Then collect and validate chunks (checks >50% success threshold, deduplicates nodes): -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\collect_chunk_results.py" ``` -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path +If exit code is 1, stop and tell the user to re-run ensuring `subagent_type="general-purpose"` is used. -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') -print(f'Cached {saved} files') -" +Save new results to cache: +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\save_results_to_cache.py" "INPUT_PATH" ``` Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\merge_semantic_results.py" +``` -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" +Clean up temp files: +```powershell +Remove-Item -Force -ErrorAction SilentlyContinue 'graphify-out\.graphify_cached.json', 'graphify-out\.graphify_uncached.txt', 'graphify-out\.graphify_semantic_new.json' ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\merge_ast_semantic.py" ``` ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path +**Before starting:** Replace `IS_DIRECTED` with `true` if `--directed` was given, otherwise `false`. Replace `INPUT_PATH` with the actual path. -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\build_graph.py" "INPUT_PATH" "IS_DIRECTED" ``` If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. -Replace INPUT_PATH with the actual path. +If this step prints `ERROR: refused to shrink`, it means the new graph has fewer nodes than the existing one. Stop and tell the user; they can force a full rebuild if the reduction is intentional. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\check_graph_health.py" "INPUT_PATH" "IS_DIRECTED" ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If the output contains `GRAPH HEALTH WARNING`, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible). ### Step 5 - Label communities Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path +**Use the Write tool** to write the labels JSON to `graphify-out/.graphify_labels_input.json` — do NOT pass labels on the command line, which breaks on long names (PowerShell command-line length limit): -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) +```json +{"0": "Community Name A", "1": "Community Name B"} +``` -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +Then run: +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\label_communities.py" "INPUT_PATH" "IS_DIRECTED" ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Replace `INPUT_PATH` and `IS_DIRECTED` as in Step 4. The script reads labels from `graphify-out/.graphify_labels_input.json` and regenerates both `graphify-out/GRAPH_REPORT.md` and `graphify-out/.graphify_labels.json`. ### Step 6 - Generate Obsidian vault (opt-in) + HTML **Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash +```powershell graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project +# or with custom dir: graphify export obsidian --dir "C:\path\to\vault" ``` Generate the HTML graph (always, unless `--no-viz`): - -```bash +```powershell graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. +These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. + +For **`--wiki`**: +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\export_wiki.py" +``` + +For **`--neo4j`** (generate Cypher file): +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\export_neo4j_cypher.py" +``` + +For **`--neo4j-push `** (push to Neo4j — ask user for credentials if not provided): +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\export_neo4j_push.py" "NEO4J_URI" "NEO4J_USER" "NEO4J_PASSWORD" +``` + +For **`--falkordb`** or **`--falkordb-push`**: see `references/exports.md` for the FalkorDB flow. + +For **`--svg`**: +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\export_svg.py" +``` + +For **`--graphml`**: +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\export_graphml.py" +``` + +For **`--mcp`**: +```powershell +& $GRAPHIFY_PYTHON -m graphify.serve graphify-out/graph.json +``` + +For the **benchmark** (when `total_words` > 5,000): +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\run_benchmark.py" +``` --- ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true +```powershell +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\save_manifest_and_cost.py" "INPUT_PATH" +Remove-Item -Force -ErrorAction SilentlyContinue ` + 'graphify-out\.graphify_detect.json', ` + 'graphify-out\.graphify_extract.json', ` + 'graphify-out\.graphify_ast.json', ` + 'graphify-out\.graphify_semantic.json', ` + 'graphify-out\.graphify_analysis.json', ` + 'graphify-out\.graphify_incremental.json', ` + 'graphify-out\.graphify_labels_input.json', ` + 'graphify-out\.needs_update' +Get-ChildItem 'graphify-out' -Filter '.graphify_chunk_*.json' -ErrorAction SilentlyContinue | Remove-Item -Force ``` Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. @@ -644,25 +461,129 @@ The graph is the map. Your job after the pipeline is to be the guide. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: +Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that the interpreter files exist. If they're missing (e.g. user deleted `graphify-out/`), re-resolve and re-save them first: -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi +```powershell +if (-not (Test-Path 'graphify-out\.graphify_python')) { + New-Item -ItemType Directory -Force -Path graphify-out | Out-Null + $py = (Get-Command python -ErrorAction SilentlyContinue) + if (-not $py) { Write-Error 'Python not found. Install graphify first: uv tool install graphifyy'; exit 1 } + & $py.Source -c "import graphify" 2>$null + if ($LASTEXITCODE -ne 0) { Write-Error 'graphify not found in Python env. Install: uv tool install graphifyy'; exit 1 } + $pyExe = (& $py.Source -c "import sys; print(sys.executable)").Trim() + $pyExe | Out-File -FilePath graphify-out\.graphify_python -Encoding utf8 -NoNewline + $scripts = (& $pyExe -c "import graphify, os; print(os.path.join(os.path.dirname(graphify.__file__), 'windows-scripts'))").Trim() + $scripts | Out-File -FilePath graphify-out\.graphify_scripts -Encoding utf8 -NoNewline +} +$GRAPHIFY_PYTHON = (Get-Content 'graphify-out\.graphify_python' -Raw).Trim() +$GRAPHIFY_SCRIPTS = (Get-Content 'graphify-out\.graphify_scripts' -Raw).Trim() ``` -## For --update and --cluster-only +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files — saves tokens and time. + +Run the interpreter guard first. Then: + +```powershell +# Save old graph for diff display after merge +Copy-Item 'graphify-out\graph.json' 'graphify-out\.graphify_old.json' -ErrorAction SilentlyContinue + +# Detect changed files (filters assets/, writes .graphify_detect.json and .graphify_incremental.json) +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\detect_incremental.py" "INPUT_PATH" +``` + +If exit code is 0 ("No files changed since last run"), stop — nothing to do. + +Check whether all changed files are code-only (no docs/papers/images = skip semantic subagents): + +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\check_code_only.py" +``` -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +The output says `code_only: True` or `code_only: False`. + +If `code_only: True`: print `[graphify update] Code-only changes — skipping semantic extraction (no LLM needed)`. Run only **Part A** (AST extraction) on the changed files, skip Part B entirely, then write an empty semantic file and go straight to Part C: + +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\ast_extraction.py" "INPUT_PATH" +& $GRAPHIFY_PYTHON -c "import json; from pathlib import Path; Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8')" +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\merge_ast_semantic.py" +``` + +If `code_only: False` (any changed file is a doc/paper/image/video): +- If any changed file is in `new_files['video']`: run Step 2.5 transcription on those files, then re-add transcripts to detect. Then run the full Steps 3A–3C pipeline as normal on the changed files. +- Otherwise: run the full Steps 3A–3C pipeline as normal on the changed files (the detect JSON is already scoped to changed files only). + +After extraction (Part C produces `graphify-out/.graphify_extract.json` with the new-files-only result), merge with the existing graph: + +```powershell +& $GRAPHIFY_PYTHON -c " +import json, sys +from graphify.build import build_merge +from pathlib import Path + +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding='utf-8')) +deleted = list(incremental.get('deleted_files', [])) + +# build_merge() reads graph.json directly (no NetworkX round-trip) so edge +# direction (calls, implements, imports) is always preserved (#801). +# prune_sources: only genuinely DELETED files. Changed/re-extracted files are +# handled by build_merge's replace-on-re-extract (#1344). +# root= relativizes absolute paths from detect_incremental so prune matches +# the graph's relative source_file values (#1361). +# directed=IS_DIRECTED: replace with True if --directed was given, else False. +G = build_merge( + [new_extraction], + graph_path='graphify-out/graph.json', + prune_sources=deleted or None, + root='INPUT_PATH', + directed=IS_DIRECTED, +) +print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') + +merged_out = { + 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], + 'edges': [ + {**{k: v for k, v in d.items() if k not in ('_src','_tgt','source','target')}, + 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + for u, v, d in G.edges(data=True) + ], + 'hyperedges': list(G.graph.get('hyperedges', [])), + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding='utf-8') +print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') +" +``` + +Replace `INPUT_PATH` and `IS_DIRECTED` as in the main pipeline. + +Then run Steps 4–9 as normal on the merged graph. + +After Step 4, show the graph diff: + +```powershell +& $GRAPHIFY_PYTHON "$GRAPHIFY_SCRIPTS\graph_diff.py" "IS_DIRECTED" +``` + +Then clean up: `Remove-Item -Force -ErrorAction SilentlyContinue 'graphify-out\.graphify_old.json'` + +--- + +## For --cluster-only + +Skip Steps 1–3. Re-run clustering on the existing graph: + +```powershell +graphify cluster-only . +``` + +`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files that Step 9 already cleaned up. When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. --- @@ -670,12 +591,14 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: -```bash +```powershell graphify query "" ``` Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +**Windows note for `save-result`:** On Windows, long answers can exceed the PowerShell command-line length limit. Use the Write tool to write the answer to `graphify-out/.graphify_answer.txt`, then pass `--answer-file graphify-out/.graphify_answer.txt` instead of `--answer "..."`. + --- ## For /graphify add and --watch @@ -701,6 +624,20 @@ If vertical scrolling breaks in PowerShell after running graphify, this is cause 3. **Reset your terminal**: close and reopen PowerShell 4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output +### Python not found / graphify import fails + +Run Step 1 manually to re-detect: +```powershell +graphify install windows +``` + +Or install manually: +```powershell +uv tool install graphifyy # preferred +# or: +pip install graphifyy +``` + --- ## Honesty Rules diff --git a/graphify/windows-scripts/add_transcripts_to_detect.py b/graphify/windows-scripts/add_transcripts_to_detect.py new file mode 100644 index 000000000..a47247c6d --- /dev/null +++ b/graphify/windows-scripts/add_transcripts_to_detect.py @@ -0,0 +1,23 @@ +import sys +import json +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +transcripts_file = Path('graphify-out/.graphify_transcripts.json') +detect_file = Path('graphify-out/.graphify_detect.json') + +if not transcripts_file.exists(): + print('No transcripts file found, skipping.') + raise SystemExit(0) + +transcript_paths = json.loads(transcripts_file.read_text(encoding='utf-8')) +if not transcript_paths: + print('No transcripts produced.') + raise SystemExit(0) + +detect = json.loads(detect_file.read_text(encoding='utf-8')) +detect.setdefault('files', {}).setdefault('docs', []).extend(transcript_paths) +detect['total_files'] = detect.get('total_files', 0) + len(transcript_paths) +detect_file.write_text(json.dumps(detect, indent=2), encoding='utf-8') +print(f'Transcribed {len(transcript_paths)} video file(s) -> treating as docs') diff --git a/graphify/windows-scripts/ast_extraction.py b/graphify/windows-scripts/ast_extraction.py new file mode 100644 index 000000000..e09f04e91 --- /dev/null +++ b/graphify/windows-scripts/ast_extraction.py @@ -0,0 +1,20 @@ +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) +input_path = detect.get('input_path', '.') +code_files = [] +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files, cache_root=Path(input_path)) + Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding='utf-8') + print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') +else: + Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding='utf-8') + print('No code files - skipping AST extraction') diff --git a/graphify/windows-scripts/build_graph.py b/graphify/windows-scripts/build_graph.py new file mode 100644 index 000000000..c046822ca --- /dev/null +++ b/graphify/windows-scripts/build_graph.py @@ -0,0 +1,47 @@ +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +input_path = sys.argv[1] if len(sys.argv) > 1 else '.' +directed = sys.argv[2].lower() == 'true' if len(sys.argv) > 2 else False + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) + +G = build_from_json(extraction, root=input_path, directed=directed) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) + +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +questions = suggest_questions(G, communities, labels) + +wrote = to_json(G, communities, 'graphify-out/graph.json') +if not wrote: + print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') + print('If this shrink is intentional (you deleted files), re-run a full build with --force.') + raise SystemExit(1) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, input_path, suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding='utf-8') +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding='utf-8') +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') diff --git a/graphify/windows-scripts/check_code_only.py b/graphify/windows-scripts/check_code_only.py new file mode 100644 index 000000000..e54c5eaa5 --- /dev/null +++ b/graphify/windows-scripts/check_code_only.py @@ -0,0 +1,12 @@ +import sys +import json +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +result = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_incremental.json').exists() else {} +code_exts = {'.py', '.ts', '.js', '.go', '.rs', '.java', '.cpp', '.c', '.rb', '.swift', '.kt', '.cs', '.scala', '.php', '.cc', '.cxx', '.hpp', '.h', '.kts', '.lua', '.toc'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) diff --git a/graphify/windows-scripts/check_extraction_cache.py b/graphify/windows-scripts/check_extraction_cache.py new file mode 100644 index 000000000..291b90a79 --- /dev/null +++ b/graphify/windows-scripts/check_extraction_cache.py @@ -0,0 +1,26 @@ +import sys +import json +from graphify.cache import check_semantic_cache +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +input_path = sys.argv[1] if len(sys.argv) > 1 else '.' +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) + +# Only content files go to semantic extraction; code is handled by AST (Part A). +# Video is transcribed to document in Step 2.5 first. +all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=Path(input_path)) + +# Always (re)write or DELETE the cache file so Part C never merges stale data (#1392). +if cached_nodes or cached_edges or cached_hyperedges: + Path('graphify-out/.graphify_cached.json').write_text( + json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), + encoding='utf-8' + ) +else: + Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) +Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding='utf-8') +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') diff --git a/graphify/windows-scripts/check_graph_exists.py b/graphify/windows-scripts/check_graph_exists.py new file mode 100644 index 000000000..99fac1c24 --- /dev/null +++ b/graphify/windows-scripts/check_graph_exists.py @@ -0,0 +1,8 @@ +import sys +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) diff --git a/graphify/windows-scripts/check_graph_health.py b/graphify/windows-scripts/check_graph_health.py new file mode 100644 index 000000000..edf6cc649 --- /dev/null +++ b/graphify/windows-scripts/check_graph_health.py @@ -0,0 +1,23 @@ +import sys, json +from pathlib import Path +from graphify.diagnostics import diagnose_extraction, format_diagnostic_report +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +input_path = sys.argv[1] if len(sys.argv) > 1 else '.' +directed = sys.argv[2].lower() == 'true' if len(sys.argv) > 2 else False + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +summary = diagnose_extraction(extraction, directed=directed, root=input_path) +print(format_diagnostic_report(summary)) +flags = [f'{summary[k]} {label}' for k, label in ( + ('dangling_endpoint_edges', 'dangling-endpoint edges'), + ('missing_endpoint_edges', 'missing-endpoint edges'), + ('self_loop_edges', 'self-loop edges'), + ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), + ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), +) if summary.get(k, 0)] +if flags: + print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.') +else: + print('Graph health: OK (no dangling/missing/collapsed edges).') diff --git a/graphify/windows-scripts/check_graphify_installed.py b/graphify/windows-scripts/check_graphify_installed.py new file mode 100644 index 000000000..e044a115b --- /dev/null +++ b/graphify/windows-scripts/check_graphify_installed.py @@ -0,0 +1,4 @@ +import sys +import graphify # exits non-zero if not installed +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') diff --git a/graphify/windows-scripts/cluster_only.py b/graphify/windows-scripts/cluster_only.py new file mode 100644 index 000000000..e0a210ac8 --- /dev/null +++ b/graphify/windows-scripts/cluster_only.py @@ -0,0 +1,37 @@ +import sys +import json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding='utf-8') +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding='utf-8') +print(f'Re-clustered: {len(communities)} communities') diff --git a/graphify/windows-scripts/collect_chunk_results.py b/graphify/windows-scripts/collect_chunk_results.py new file mode 100644 index 000000000..feea6e450 --- /dev/null +++ b/graphify/windows-scripts/collect_chunk_results.py @@ -0,0 +1,58 @@ +import sys +import json, sys +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +out_dir = Path('graphify-out') +chunk_files = sorted(out_dir.glob('.graphify_chunk_*.json')) + +if not chunk_files: + print('ERROR: No chunk files found in graphify-out/. Subagents may not have written results.') + print('Re-run and ensure subagent_type="general-purpose" is used.') + sys.exit(1) + +all_nodes, all_edges, all_hyperedges = [], [], [] +total_input = total_output = 0 +failed = [] + +for cf in chunk_files: + try: + data = json.loads(cf.read_text(encoding='utf-8')) + if 'nodes' not in data or 'edges' not in data: + raise ValueError('missing nodes or edges keys') + all_nodes.extend(data['nodes']) + all_edges.extend(data['edges']) + all_hyperedges.extend(data.get('hyperedges', [])) + total_input += data.get('input_tokens', 0) + total_output += data.get('output_tokens', 0) + except Exception as e: + print(f'WARNING: {cf.name} failed validation: {e}') + failed.append(cf.name) + +total = len(chunk_files) +if len(failed) > total / 2: + print(f'ERROR: {len(failed)}/{total} chunks failed. Re-run and ensure subagent_type="general-purpose".') + sys.exit(1) + +if failed: + print(f'WARNING: {len(failed)} chunk(s) invalid: {", ".join(failed)}') + +seen: set = set() +deduped_nodes = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped_nodes.append(n) + +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': deduped_nodes, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': total_input, + 'output_tokens': total_output, +}, indent=2), encoding='utf-8') +print(f'Collected {total - len(failed)}/{total} chunks: {len(deduped_nodes)} nodes, {len(all_edges)} edges') + +for cf in chunk_files: + cf.unlink(missing_ok=True) diff --git a/graphify/windows-scripts/detect_files.py b/graphify/windows-scripts/detect_files.py new file mode 100644 index 000000000..139196cc9 --- /dev/null +++ b/graphify/windows-scripts/detect_files.py @@ -0,0 +1,20 @@ +import sys, json +from graphify.detect import detect +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +def _is_assets(path_str: str) -> bool: + return any(part.lower() == 'assets' for part in Path(path_str).parts) + +input_path = Path(sys.argv[1]).resolve() +result = detect(input_path) +result['input_path'] = str(input_path) + +# Discard files inside any folder named 'assets' +for category in result.get('files', {}): + result['files'][category] = [f for f in result['files'][category] if not _is_assets(f)] +result['total_files'] = sum(len(v) for v in result.get('files', {}).values()) + +Path('graphify-out').mkdir(exist_ok=True) +Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding='utf-8') diff --git a/graphify/windows-scripts/detect_incremental.py b/graphify/windows-scripts/detect_incremental.py new file mode 100644 index 000000000..f4962d11a --- /dev/null +++ b/graphify/windows-scripts/detect_incremental.py @@ -0,0 +1,39 @@ +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +def _is_assets(path_str: str) -> bool: + return any(part.lower() == 'assets' for part in Path(path_str).parts) + +result = detect_incremental(Path(sys.argv[1])) + +# Discard files inside any folder named 'assets' +for category in result.get('files', {}): + result['files'][category] = [f for f in result['files'][category] if not _is_assets(f)] +for category in result.get('new_files', {}): + result['new_files'][category] = [f for f in result['new_files'][category] if not _is_assets(f)] +result['new_total'] = sum(len(v) for v in result.get('new_files', {}).values()) +result['total_files'] = sum(len(v) for v in result.get('files', {}).values()) + +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding='utf-8') +if new_total == 0: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +print(f'{new_total} new/changed file(s) to re-extract.') + +# Write .graphify_detect.json scoped to new files only so all downstream +# scripts (check_extraction_cache, print_timing_estimate, build_graph, etc.) +# operate on the changed file set, not the full corpus. +detect_for_update = { + 'files': result['new_files'], + 'all_files': result.get('files', {}), # full corpus for manifest-saving + 'total_files': result['new_total'], + 'total_words': result.get('total_words', 0), + 'skipped_sensitive': result.get('skipped_sensitive', []), + 'input_path': result.get('input_path', str(Path(sys.argv[1]).resolve())), +} +Path('graphify-out/.graphify_detect.json').write_text(json.dumps(detect_for_update, ensure_ascii=False), encoding='utf-8') diff --git a/graphify/windows-scripts/explain_node.py b/graphify/windows-scripts/explain_node.py new file mode 100644 index 000000000..033a9dd59 --- /dev/null +++ b/graphify/windows-scripts/explain_node.py @@ -0,0 +1,37 @@ +import sys, json +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +term = sys.argv[1] if len(sys.argv) > 1 else '' +term_lower = term.lower() + +data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) +G = json_graph.node_link_graph(data, edges='links') + +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label', '').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get("label", nid)}') +print(f' source: {data_n.get("source_file", "unknown")}') +print(f' type: {data_n.get("file_type", "unknown")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + edge = G.edges[nid, neighbor] + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') diff --git a/graphify/windows-scripts/export_graphml.py b/graphify/windows-scripts/export_graphml.py new file mode 100644 index 000000000..ac50a6aca --- /dev/null +++ b/graphify/windows-scripts/export_graphml.py @@ -0,0 +1,16 @@ +import sys +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') diff --git a/graphify/windows-scripts/export_html.py b/graphify/windows-scripts/export_html.py new file mode 100644 index 000000000..6ccf973a8 --- /dev/null +++ b/graphify/windows-scripts/export_html.py @@ -0,0 +1,21 @@ +import sys +import json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') diff --git a/graphify/windows-scripts/export_neo4j_cypher.py b/graphify/windows-scripts/export_neo4j_cypher.py new file mode 100644 index 000000000..3053397e1 --- /dev/null +++ b/graphify/windows-scripts/export_neo4j_cypher.py @@ -0,0 +1,11 @@ +import sys +import json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8'))) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') diff --git a/graphify/windows-scripts/export_neo4j_push.py b/graphify/windows-scripts/export_neo4j_push.py new file mode 100644 index 000000000..42d7b1be2 --- /dev/null +++ b/graphify/windows-scripts/export_neo4j_push.py @@ -0,0 +1,19 @@ +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +uri = sys.argv[1] if len(sys.argv) > 1 else 'bolt://localhost:7687' +user = sys.argv[2] if len(sys.argv) > 2 else 'neo4j' +password = sys.argv[3] if len(sys.argv) > 3 else '' + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri=uri, user=user, password=password, communities=communities) +print(f'Pushed to Neo4j: {result["nodes"]} nodes, {result["edges"]} edges') diff --git a/graphify/windows-scripts/export_obsidian.py b/graphify/windows-scripts/export_obsidian.py new file mode 100644 index 000000000..f92e21428 --- /dev/null +++ b/graphify/windows-scripts/export_obsidian.py @@ -0,0 +1,28 @@ +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +obsidian_dir = sys.argv[1] if len(sys.argv) > 1 else 'graphify-out/obsidian' + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, obsidian_dir, community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in {obsidian_dir}/') + +to_canvas(G, communities, f'{obsidian_dir}/graph.canvas', community_labels=labels or None) +print(f'Canvas: {obsidian_dir}/graph.canvas - open in Obsidian for structured community layout') +print() +print(f'Open {obsidian_dir}/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') diff --git a/graphify/windows-scripts/export_svg.py b/graphify/windows-scripts/export_svg.py new file mode 100644 index 000000000..0beec0321 --- /dev/null +++ b/graphify/windows-scripts/export_svg.py @@ -0,0 +1,22 @@ +import sys +import json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +try: + to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) + print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +except ImportError as e: + print(f'SVG export skipped: {e}') + raise SystemExit(1) diff --git a/graphify/windows-scripts/export_wiki.py b/graphify/windows-scripts/export_wiki.py new file mode 100644 index 000000000..3a796e608 --- /dev/null +++ b/graphify/windows-scripts/export_wiki.py @@ -0,0 +1,22 @@ +import sys +import json +from graphify.build import build_from_json +from graphify.wiki import to_wiki +from graphify.analyze import god_nodes +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) +labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} +gods = god_nodes(G) + +n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) +print(f'Wiki: {n} articles written to graphify-out/wiki/') +print(' graphify-out/wiki/index.md -> agent entry point') diff --git a/graphify/windows-scripts/graph_diff.py b/graphify/windows-scripts/graph_diff.py new file mode 100644 index 000000000..94ecb8421 --- /dev/null +++ b/graphify/windows-scripts/graph_diff.py @@ -0,0 +1,22 @@ +import sys +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_old.json').exists() else None +new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) diff --git a/graphify/windows-scripts/ingest_url.py b/graphify/windows-scripts/ingest_url.py new file mode 100644 index 000000000..b53f8001f --- /dev/null +++ b/graphify/windows-scripts/ingest_url.py @@ -0,0 +1,19 @@ +import sys +from graphify.ingest import ingest +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +url = sys.argv[1] if len(sys.argv) > 1 else '' +author = sys.argv[2] if len(sys.argv) > 2 else None +contributor = sys.argv[3] if len(sys.argv) > 3 else None + +try: + out = ingest(url, Path('./raw'), author=author, contributor=contributor) + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) diff --git a/graphify/windows-scripts/label_communities.py b/graphify/windows-scripts/label_communities.py new file mode 100644 index 000000000..eca0f9c0e --- /dev/null +++ b/graphify/windows-scripts/label_communities.py @@ -0,0 +1,37 @@ +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import suggest_questions +from graphify.report import generate +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +input_path = sys.argv[1] if len(sys.argv) > 1 else '.' +directed = sys.argv[2].lower() == 'true' if len(sys.argv) > 2 else False + +# Read labels from file to avoid shell command-line length limits. +# Falls back to inline sys.argv[3] for backward compatibility. +_labels_file = Path('graphify-out/.graphify_labels_input.json') +if _labels_file.exists(): + labels = {int(k): v for k, v in json.loads(_labels_file.read_text(encoding='utf-8')).items()} +elif len(sys.argv) > 3: + labels = {int(k): v for k, v in json.loads(sys.argv[3]).items()} +else: + labels = {} + +extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) +analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding='utf-8')) + +G = build_from_json(extraction, root=input_path, directed=directed) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, input_path, suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding='utf-8') +Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding='utf-8') +print('Report updated with community labels') diff --git a/graphify/windows-scripts/merge_ast_semantic.py b/graphify/windows-scripts/merge_ast_semantic.py new file mode 100644 index 000000000..4a3118707 --- /dev/null +++ b/graphify/windows-scripts/merge_ast_semantic.py @@ -0,0 +1,39 @@ +import sys +import json +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding='utf-8')) +sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding='utf-8')) + +# Semantic nodes take priority: richer labels, rationale, cross-file context. +# AST source_location is backfilled onto semantic nodes that lack it so precise +# line numbers are never lost. +ast_by_id = {n['id']: n for n in ast['nodes']} +seen: set = set() +merged_nodes = [] +for n in sem['nodes']: + seen.add(n['id']) + ast_node = ast_by_id.get(n['id']) + if ast_node and not n.get('source_location') and ast_node.get('source_location'): + n = dict(n, source_location=ast_node['source_location']) + merged_nodes.append(n) +for n in ast['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding='utf-8') +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast["nodes"])} AST + {len(sem["nodes"])} semantic)') diff --git a/graphify/windows-scripts/merge_graphs.py b/graphify/windows-scripts/merge_graphs.py new file mode 100644 index 000000000..475a04154 --- /dev/null +++ b/graphify/windows-scripts/merge_graphs.py @@ -0,0 +1,49 @@ +import sys +import json +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +# Load existing graph.json (NetworkX node-link format: nodes + links keys) +existing_data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) +existing_nodes = existing_data.get('nodes', []) +# NetworkX serialises edges as 'links'; also handle 'edges' for compatibility +existing_edges = existing_data.get('links', existing_data.get('edges', [])) +existing_hyperedges = existing_data.get('hyperedges', []) + +# Load new extraction (changed files only, produced by Step 3) +new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +new_nodes = new_extraction.get('nodes', []) +new_edges = new_extraction.get('edges', []) +new_hyperedges = new_extraction.get('hyperedges', []) + +# Merge nodes: new/changed versions take priority; existing nodes fill the rest +new_ids = {n['id'] for n in new_nodes} +merged_nodes = list(new_nodes) +for n in existing_nodes: + if n.get('id') not in new_ids: + merged_nodes.append(n) + +# Merge edges: new edges take priority over existing edges for the same pair +new_edge_pairs = {(e.get('source'), e.get('target')) for e in new_edges} +merged_edges = list(new_edges) +for e in existing_edges: + if (e.get('source'), e.get('target')) not in new_edge_pairs: + merged_edges.append(e) + +# Merge hyperedges: new hyperedges take priority by id +new_he_ids = {h.get('id') for h in new_hyperedges} +merged_hyperedges = list(new_hyperedges) + [ + h for h in existing_hyperedges if h.get('id') not in new_he_ids +] + +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': new_extraction.get('input_tokens', 0), + 'output_tokens': new_extraction.get('output_tokens', 0), +} + +Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding='utf-8') +print(f'Merged: {len(merged_nodes)} nodes, {len(merged_edges)} edges ({len(existing_nodes)} existing + {len(new_nodes)} new/changed)') diff --git a/graphify/windows-scripts/merge_semantic_results.py b/graphify/windows-scripts/merge_semantic_results.py new file mode 100644 index 000000000..4d3652ff1 --- /dev/null +++ b/graphify/windows-scripts/merge_semantic_results.py @@ -0,0 +1,28 @@ +import sys +import json +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes': [], 'edges': [], 'hyperedges': []} +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes': [], 'edges': [], 'hyperedges': []} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding='utf-8') +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached["nodes"])} from cache, {len(new.get("nodes", []))} new)') diff --git a/graphify/windows-scripts/path_between_nodes.py b/graphify/windows-scripts/path_between_nodes.py new file mode 100644 index 000000000..da529bca2 --- /dev/null +++ b/graphify/windows-scripts/path_between_nodes.py @@ -0,0 +1,45 @@ +import sys, json +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +a_term = sys.argv[1] if len(sys.argv) > 1 else '' +b_term = sys.argv[2] if len(sys.argv) > 2 else '' + +data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) +G = json_graph.node_link_graph(data, edges='links') + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label', '').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + edge = G.edges[nid, path[i+1]] + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') diff --git a/graphify/windows-scripts/print_detect_summary.py b/graphify/windows-scripts/print_detect_summary.py new file mode 100644 index 000000000..8555291c0 --- /dev/null +++ b/graphify/windows-scripts/print_detect_summary.py @@ -0,0 +1,54 @@ +import sys +import json, sys +from collections import Counter +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) +files = detect.get('files', {}) +total_files = detect.get('total_files', 0) +total_words = detect.get('total_words', 0) +skipped = detect.get('skipped_sensitive', []) +input_path = Path(detect.get('input_path', '.')) + +if total_files == 0: + print('No supported files found.') + sys.exit(1) + +print(f'Corpus: {total_files} files - ~{total_words:,} words') +for cat in ['code', 'docs', 'document', 'papers', 'paper', 'images', 'image', 'video']: + file_list = [ + f for f in files.get(cat, []) + if '/assets/' not in f and '\\assets\\' not in f + ] + if not file_list: + continue + display = cat if cat not in ('document', 'paper', 'image') else {'document': 'docs', 'paper': 'papers', 'image': 'images'}[cat] + exts = sorted({Path(f).suffix for f in file_list if Path(f).suffix}) + pad = max(1, 10 - len(display)) + print(f' {display}:{" " * pad}{len(file_list)} files ({" ".join(exts[:6])})') + +if skipped: + print(f' [{len(skipped)} sensitive file(s) skipped]') + +has_video = bool([f for f in files.get('video', []) if '/assets/' not in f and '\\assets\\' not in f]) +print(f'HAS_VIDEO={str(has_video).lower()}') + +if total_words > 2_000_000 or total_files > 200: + print(f'\nWARNING: Large corpus ({total_files} files, ~{total_words:,} words). Recommend running on a subfolder.') + subdir_counts: Counter = Counter() + all_file_paths = [f for flist in files.values() for f in flist] + for f in all_file_paths: + try: + rel = Path(f).relative_to(input_path) + top = rel.parts[0] if len(rel.parts) > 1 else '.' + except ValueError: + top = Path(f).parts[-2] if len(Path(f).parts) > 1 else '.' + subdir_counts[top] += 1 + print('Top subdirectories by file count:') + for subdir, count in subdir_counts.most_common(5): + print(f' {subdir}/ ({count} files)') + sys.exit(2) + +sys.exit(0) diff --git a/graphify/windows-scripts/print_timing_estimate.py b/graphify/windows-scripts/print_timing_estimate.py new file mode 100644 index 000000000..e758570aa --- /dev/null +++ b/graphify/windows-scripts/print_timing_estimate.py @@ -0,0 +1,20 @@ +import sys +import json, math +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) +files = detect.get('files', {}) + +non_code = sum(len(v) for k, v in files.items() if k != 'code') + +if non_code == 0: + print('Code-only corpus - no semantic extraction needed.') + raise SystemExit(0) + +agents = math.ceil(non_code / 22) +parallel_limit = 5 +estimated_secs = math.ceil(agents / parallel_limit) * 45 + +print(f'Semantic extraction: ~{non_code} files -> {agents} agents, estimated ~{estimated_secs}s') diff --git a/graphify/windows-scripts/query_graph.py b/graphify/windows-scripts/query_graph.py new file mode 100644 index 000000000..32533f500 --- /dev/null +++ b/graphify/windows-scripts/query_graph.py @@ -0,0 +1,79 @@ +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +question = sys.argv[1] if len(sys.argv) > 1 else '' +mode = sys.argv[2] if len(sys.argv) > 2 else 'bfs' +token_budget = int(sys.argv[3]) if len(sys.argv) > 3 else 2000 + +data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) +G = json_graph.node_link_graph(data, edges='links') + +terms = [t.lower() for t in question.split() if len(t) > 3] + +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +char_budget = token_budget * 4 + +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get("label", n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get("label", nid)} [src={d.get("source_file", "")} loc={d.get("source_location", "")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + d = G.edges[u, v] + lines.append(f' EDGE {G.nodes[u].get("label", u)} --{d.get("relation", "")} [{d.get("confidence", "")}]--> {G.nodes[v].get("label", v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) diff --git a/graphify/windows-scripts/run_benchmark.py b/graphify/windows-scripts/run_benchmark.py new file mode 100644 index 000000000..b005a8ead --- /dev/null +++ b/graphify/windows-scripts/run_benchmark.py @@ -0,0 +1,14 @@ +import sys +import json, sys +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) +total_words = detection.get('total_words', 0) +if total_words <= 5000: + sys.exit(0) + +result = run_benchmark('graphify-out/graph.json', corpus_words=total_words) +print_benchmark(result) diff --git a/graphify/windows-scripts/save_manifest_and_cost.py b/graphify/windows-scripts/save_manifest_and_cost.py new file mode 100644 index 000000000..e37aa19c2 --- /dev/null +++ b/graphify/windows-scripts/save_manifest_and_cost.py @@ -0,0 +1,38 @@ +import sys +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +input_path = sys.argv[1] if len(sys.argv) > 1 else '.' +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) + +# In --update mode, 'all_files' carries the full corpus; 'files' is the changed subset. +# Full-rebuild mode populates only 'files', so the fallback handles that. +# root= relativizes manifest keys so a later --update matches cached files (#1417). +save_manifest(detect.get('all_files') or detect['files'], root=Path(input_path)) + +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text(encoding='utf-8')) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding='utf-8') + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost["total_input_tokens"]:,} input, {cost["total_output_tokens"]:,} output ({len(cost["runs"])} runs)') diff --git a/graphify/windows-scripts/save_results_to_cache.py b/graphify/windows-scripts/save_results_to_cache.py new file mode 100644 index 000000000..7c2592071 --- /dev/null +++ b/graphify/windows-scripts/save_results_to_cache.py @@ -0,0 +1,11 @@ +import sys +import json +from graphify.cache import save_semantic_cache +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +input_path = sys.argv[1] if len(sys.argv) > 1 else '.' +new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding='utf-8')) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=Path(input_path)) +print(f'Cached {saved} files') diff --git a/graphify/windows-scripts/transcribe_video.py b/graphify/windows-scripts/transcribe_video.py new file mode 100644 index 000000000..6c8e5755e --- /dev/null +++ b/graphify/windows-scripts/transcribe_video.py @@ -0,0 +1,13 @@ +import sys +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) diff --git a/graphify/windows-scripts/write_python_path.py b/graphify/windows-scripts/write_python_path.py new file mode 100644 index 000000000..88c9d8abe --- /dev/null +++ b/graphify/windows-scripts/write_python_path.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(encoding='utf-8') + +Path('graphify-out').mkdir(exist_ok=True) +Path('graphify-out/.graphify_python').write_text(sys.executable, encoding='utf-8') From 7ff8e48e50d2f0e7356178506b5fe854e418f5a1 Mon Sep 17 00:00:00 2001 From: antonioscarinci <96891485+antonioscarinci@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:36:55 +0200 Subject: [PATCH 2/3] fix: pyproject Fix the pyproject.toml so that the Windows skill .py scripts are installed --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4ea71556b..faaf75c48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ include-package-data = false # under graphify/skills//references/, and the always-on injection blocks # under graphify/always_on/. There is no graphify/skills//SKILL.md in the # repo, so no SKILL.md glob is needed here. -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-agents.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md"] +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-agents.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md", "windows-scripts/*.py"] [tool.pytest.ini_options] testpaths = ["tests"] From 4e297aa91245a6fcab1cb63cfdcbc6d47cb78f59 Mon Sep 17 00:00:00 2001 From: antonioscarinci <96891485+antonioscarinci@users.noreply.github.com> Date: Sun, 28 Jun 2026 03:41:34 +0200 Subject: [PATCH 3/3] fix(update): prevent full re-extraction when --update is called MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three root causes fixed: 1. skill-windows.md: add explicit routing guard so an AI following 'do not skip steps' jumps directly to ## For --update after Step 1 instead of running Steps 2-3 (full detect + full extraction) first. Also add note after build_merge() block confirming Steps 2-3 must not be re-run. 2. detect_incremental.py: pass kind='ast' to detect_incremental() as documented — kind='semantic' (default) mismatches when a prior run used kind='ast', treating every previously-updated file as changed. 3. save_manifest_and_cost.py: pass detect['files'] (changed files only) instead of detect.get('all_files') (full corpus). save_manifest() seeds unchanged entries from the existing manifest, so re-hashing the entire corpus on every --update is unnecessary and reads every source file. The installed bash skill's references/update.md gets the same fix (incremental['new_files'] instead of incremental['files']). Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01GLua3wapoQZo5Z4ejhc5qt --- graphify/skill-windows.md | 8 +++++++- graphify/windows-scripts/detect_incremental.py | 2 +- graphify/windows-scripts/save_manifest_and_cost.py | 7 ++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 5d9f38fb3..ec3974715 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -56,6 +56,12 @@ If no path was given, use `.` (current directory). Do not ask the user for a pat If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Command routing — read before running any step:** +- `--update`: Run only Step 1 (interpreter guard), then jump to `## For --update (incremental re-extraction)`. **Do not run Steps 2–3** — they do a full file scan and full extraction, which `--update` deliberately avoids to save tokens and time. +- `--cluster-only`: Run Step 1, then jump to `## For --cluster-only`. +- `--watch` or `/graphify add`: Run Step 1, then see `references/add-watch.md`. +- All other invocations: follow Steps 0–9 below in order. + Follow these steps in order. Do not skip steps. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) @@ -563,7 +569,7 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) Replace `INPUT_PATH` and `IS_DIRECTED` as in the main pipeline. -Then run Steps 4–9 as normal on the merged graph. +Then run Steps 4–9 as normal. The merged `.graphify_extract.json` and scoped `.graphify_detect.json` are already written above — **do not re-run Steps 2–3**. After Step 4, show the graph diff: diff --git a/graphify/windows-scripts/detect_incremental.py b/graphify/windows-scripts/detect_incremental.py index f4962d11a..8529c7938 100644 --- a/graphify/windows-scripts/detect_incremental.py +++ b/graphify/windows-scripts/detect_incremental.py @@ -7,7 +7,7 @@ def _is_assets(path_str: str) -> bool: return any(part.lower() == 'assets' for part in Path(path_str).parts) -result = detect_incremental(Path(sys.argv[1])) +result = detect_incremental(Path(sys.argv[1]), kind='ast') # Discard files inside any folder named 'assets' for category in result.get('files', {}): diff --git a/graphify/windows-scripts/save_manifest_and_cost.py b/graphify/windows-scripts/save_manifest_and_cost.py index e37aa19c2..6eb0a5a7d 100644 --- a/graphify/windows-scripts/save_manifest_and_cost.py +++ b/graphify/windows-scripts/save_manifest_and_cost.py @@ -9,10 +9,11 @@ input_path = sys.argv[1] if len(sys.argv) > 1 else '.' detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding='utf-8')) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed subset. -# Full-rebuild mode populates only 'files', so the fallback handles that. +# 'files' is the changed subset in --update mode, or the full corpus in a full build. +# save_manifest() seeds unchanged entries from the existing manifest, so passing +# only changed files is correct — there is no need to re-hash the full corpus. # root= relativizes manifest keys so a later --update matches cached files (#1417). -save_manifest(detect.get('all_files') or detect['files'], root=Path(input_path)) +save_manifest(detect['files'], root=Path(input_path)) extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding='utf-8')) input_tok = extract.get('input_tokens', 0)