From a7a63d8e0eaa8040efcd1f1e94c940d94b89cdec Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Wed, 26 Aug 2026 01:55:14 +0530 Subject: [PATCH] fix(skill): stamp _origin on the semantic tier so update cannot delete it (#2843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build._is_ast_tier reads `_origin` when present and otherwise guesses the tier from the SHAPE of source_location: 'L' means AST. extract() stamps its own items 'ast'; nothing stamped the semantic side. A subagent (or a backend) that reports 'L12' for a document section therefore made that node read as AST, and the next `graphify update` — which replaces the AST tier of every re-extracted code file — deleted the document's whole semantic layer. On the reporter's 21-document corpus: 8 files, 133 nodes and 236 edges lost on a commit that touched one file. Three sites now say so explicitly, with setdefault so an existing stamp is never overwritten: * the runbook's Part C merge (core, aider and devin fragments) stamps every semantic node and edge before merging — the block the reporter's run went through; * llm.extract_corpus_parallel stamps its result (_stamp_semantic_origin), so library callers and the CLI's fresh results carry it; * the CLI's AST+semantic merge stamps sem_result as a whole, which also covers semantic-cache hits written before this change. The skillgen artifacts are regenerated, the expected/ snapshots blessed, and the new Part C lines registered as a sanctioned monolith diff. --- graphify/cli.py | 8 + graphify/llm.py | 19 +++ graphify/skill-agents.md | 12 ++ graphify/skill-aider.md | 12 ++ graphify/skill-amp.md | 12 ++ graphify/skill-claw.md | 12 ++ graphify/skill-codex.md | 12 ++ graphify/skill-copilot.md | 12 ++ graphify/skill-devin.md | 12 ++ graphify/skill-droid.md | 12 ++ graphify/skill-kilo.md | 12 ++ graphify/skill-kiro.md | 12 ++ graphify/skill-opencode.md | 12 ++ graphify/skill-pi.md | 12 ++ graphify/skill-trae.md | 12 ++ graphify/skill-vscode.md | 12 ++ graphify/skill-windows.md | 12 ++ graphify/skill.md | 12 ++ tests/test_semantic_origin_stamp.py | 142 ++++++++++++++++++ .../expected/graphify__skill-agents.md | 12 ++ .../expected/graphify__skill-aider.md | 12 ++ .../skillgen/expected/graphify__skill-amp.md | 12 ++ .../skillgen/expected/graphify__skill-claw.md | 12 ++ .../expected/graphify__skill-codex.md | 12 ++ .../expected/graphify__skill-copilot.md | 12 ++ .../expected/graphify__skill-devin.md | 12 ++ .../expected/graphify__skill-droid.md | 12 ++ .../skillgen/expected/graphify__skill-kilo.md | 12 ++ .../skillgen/expected/graphify__skill-kiro.md | 12 ++ .../expected/graphify__skill-opencode.md | 12 ++ tools/skillgen/expected/graphify__skill-pi.md | 12 ++ .../skillgen/expected/graphify__skill-trae.md | 12 ++ .../expected/graphify__skill-vscode.md | 12 ++ .../expected/graphify__skill-windows.md | 12 ++ tools/skillgen/expected/graphify__skill.md | 12 ++ tools/skillgen/fragments/core/aider.md | 12 ++ tools/skillgen/fragments/core/core.md | 12 ++ tools/skillgen/fragments/core/devin.md | 12 ++ tools/skillgen/gen.py | 30 ++++ 39 files changed, 619 insertions(+) create mode 100644 tests/test_semantic_origin_stamp.py diff --git a/graphify/cli.py b/graphify/cli.py index bd2956815b..8fc00cacd9 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -3898,6 +3898,14 @@ def _progress(idx: int, total: int, _result: dict) -> None: # first means semantic node attributes win on collision (richer labels # for symbols also referenced in docs). Hyperedges only come from the # semantic side. + # Everything in sem_result is semantic tier - fresh results and cache + # hits alike - so say so before the merge (#2843): an unstamped item + # is classified by the shape of its source_location, and a doc node + # located at 'L12' would read as AST and be deleted by the next + # incremental rebuild. Fresh results already carry the stamp + # (llm._stamp_semantic_origin); cache entries from before it do not. + from graphify.llm import _stamp_semantic_origin as _stamp_sem + _stamp_sem(sem_result) merged: dict = { "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), diff --git a/graphify/llm.py b/graphify/llm.py index ae2119773a..b66cb9e3d8 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -2794,6 +2794,7 @@ def _out_of_scope(item: dict) -> bool: if p.resolve() not in {c.resolve() for c in covered} ) merged["uncovered_files"] = [str(p) for p in uncovered] + _stamp_semantic_origin(merged) if uncovered: shown = ", ".join(p.name for p in uncovered[:5]) more = f" (+{len(uncovered) - 5} more)" if len(uncovered) > 5 else "" @@ -2806,6 +2807,24 @@ def _out_of_scope(item: dict) -> bool: return merged +def _stamp_semantic_origin(result: dict) -> dict: + """Mark every node and edge of a semantic extraction ``_origin: "semantic"``. + + build._is_ast_tier reads ``_origin`` when present and otherwise guesses + from the shape of ``source_location``: ``L`` means AST. A backend + or subagent that reports line numbers for a document's sections made + those nodes read as AST, and the next incremental rebuild deleted them + with the re-extracted code file they seemed to belong to (#2843). + extract() stamps its items ``ast``; this is the semantic counterpart. + Existing stamps are kept (``setdefault``); returns ``result``. + """ + for key in ("nodes", "edges"): + for item in result.get(key, []) or []: + if isinstance(item, dict): + item.setdefault("_origin", "semantic") + return result + + def _merge_into(merged: dict, result: dict) -> None: """Append a chunk result into the running merged accumulator.""" merged["nodes"].extend(result.get("nodes", [])) diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 190827d9ac..523beddb5f 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 4996beb787..80f8746aae 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -361,6 +361,18 @@ from pathlib import Path ast = json.loads(Path('.graphify_ast.json').read_text()) sem = json.loads(Path('.graphify_semantic.json').read_text()) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 190827d9ac..523beddb5f 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index abd2811d23..d4d86f3ed1 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index af3f723c78..8a4cce219c 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index abd2811d23..d4d86f3ed1 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index f9be846cbf..0cd4705ebf 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -425,6 +425,18 @@ from graphify.semantic_cleanup import sanitize_semantic_fragment ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index fd148d485d..ffc757e631 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 3e70b050a4..e0782dd69d 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index abd2811d23..d4d86f3ed1 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 91ced60675..64c62f3de0 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -358,6 +358,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index abd2811d23..d4d86f3ed1 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 050667bc20..9279b32c2e 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -364,6 +364,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 20c7c0835c..d91fa12872 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -362,6 +362,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index b09ecca3c4..5eb98aba3a 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -393,6 +393,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding="utf-8")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding="utf-8")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/graphify/skill.md b/graphify/skill.md index abd2811d23..d4d86f3ed1 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tests/test_semantic_origin_stamp.py b/tests/test_semantic_origin_stamp.py new file mode 100644 index 0000000000..303e4b3e56 --- /dev/null +++ b/tests/test_semantic_origin_stamp.py @@ -0,0 +1,142 @@ +"""Semantic-tier items must carry `_origin`, never be guessed by shape (#2843). + +`build._is_ast_tier` reads `_origin` when present and otherwise falls back to +the shape of `source_location`: `L` means AST. extract() stamps its own +items `ast`; nothing stamped the semantic side. A subagent (or a backend) +that reports `L12` for a document section therefore made that node read as +AST, and the next `graphify update` — which replaces the AST tier of every +re-extracted code file — deleted the document's whole semantic layer. On the +reporter's 21-document corpus: 8 files, 133 nodes, 236 edges lost on a +commit that touched one file. +""" +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +from graphify.build import _is_ast_tier + +try: + from graphify.llm import _stamp_semantic_origin +except ImportError: # pre-fix tree + _stamp_semantic_origin = None + +FRAGMENTS = Path(__file__).resolve().parent.parent / "tools" / "skillgen" / "fragments" / "core" + + +def _line_located_doc_node(**extra) -> dict: + """What a subagent writes for a section of a document without section + numbers: a line-number location, exactly the AST shape.""" + return {"id": "docs_guide_deploy", "label": "Deploy", "file_type": "document", + "source_file": "docs/guide.md", "source_location": "L12", **extra} + + +def test_the_shape_fallback_is_the_trap(): + """Unstamped, a line-located document node reads as AST.""" + assert _is_ast_tier(_line_located_doc_node()) is True + assert _is_ast_tier(_line_located_doc_node(_origin="semantic")) is False + + +@pytest.mark.skipif(_stamp_semantic_origin is None, reason="pre-fix tree") +def test_stamping_marks_nodes_and_edges_and_keeps_existing_marks(): + result = { + "nodes": [_line_located_doc_node(), {"id": "x", "_origin": "ast"}], + "edges": [{"source": "a", "target": "b", "relation": "references", "source_location": "L3"}], + "hyperedges": [{"nodes": ["a", "b"]}], + } + out = _stamp_semantic_origin(result) + assert out is result + assert result["nodes"][0]["_origin"] == "semantic" + assert result["nodes"][1]["_origin"] == "ast" # never overwritten + assert result["edges"][0]["_origin"] == "semantic" + assert not _is_ast_tier(result["nodes"][0]) + assert not _is_ast_tier(result["edges"][0]) + assert "_origin" not in result["hyperedges"][0] # hyperedges are semantic by construction + + +# --------------------------------------------------------------------------- +# The runbook's Part C — the block the reporter's run went through +# --------------------------------------------------------------------------- + +def _part_c_python(fragment: Path) -> str: + """The python passed to `-c` in the fragment's Part C block, unescaped the + way the shell would hand it to the interpreter.""" + text = fragment.read_text(encoding="utf-8") + start = text.index("#### Part C - Merge AST + semantic into final extraction") + block = text[start:] + m = re.search(r'-c "\n(.*?)\n"\n```', block, re.S) + assert m, f"no Part C python block in {fragment.name}" + return m.group(1).replace('\\"', '"') + + +@pytest.mark.parametrize("fragment", ["core.md", "aider.md", "devin.md"]) +def test_part_c_stamps_the_semantic_side(fragment, tmp_path): + code = _part_c_python(FRAGMENTS / fragment) + # The three runbooks read their intermediates from different places. + ast_path = re.search(r"Path\('([^']*\.graphify_ast\.json)'\)", code).group(1) + sem_path = re.search(r"Path\('([^']*\.graphify_semantic\.json)'\)", code).group(1) + out_path = re.search(r"Path\('([^']*\.graphify_extract\.json)'\)", code).group(1) + for rel in (ast_path, sem_path, out_path): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / "graphify-out").mkdir(exist_ok=True) + ast = {"nodes": [{"id": "src_app_main", "label": "main", "file_type": "code", + "source_file": "src/app.py", "source_location": "L1", "_origin": "ast"}], + "edges": []} + sem = {"nodes": [_line_located_doc_node()], + "edges": [{"source": "docs_guide_deploy", "target": "src_app_main", + "relation": "references", "source_file": "docs/guide.md", + "source_location": "L12", "confidence": "EXTRACTED"}], + "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + (tmp_path / ast_path).write_text(json.dumps(ast), encoding="utf-8") + (tmp_path / sem_path).write_text(json.dumps(sem), encoding="utf-8") + r = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, capture_output=True, text=True) + assert r.returncode == 0, r.stderr + merged = json.loads((tmp_path / out_path).read_text(encoding="utf-8")) + by_id = {n["id"]: n for n in merged["nodes"]} + assert by_id["src_app_main"]["_origin"] == "ast" + assert by_id["docs_guide_deploy"]["_origin"] == "semantic" + assert all(e["_origin"] == "semantic" for e in merged["edges"] if e["source_file"] == "docs/guide.md") + assert not _is_ast_tier(by_id["docs_guide_deploy"]) + + +# --------------------------------------------------------------------------- +# The consequence: an incremental rebuild keeps the document's layer +# --------------------------------------------------------------------------- + +def test_a_stamped_document_survives_the_code_files_re_extraction(tmp_path): + """The failure the reporter measured: re-extract one code file, and the + line-located document node — unstamped — is treated as that file's stale + AST and dropped. Stamped, it stays.""" + from graphify.build import build_from_json, build_merge + from graphify.export import to_json + + def graph_with(doc_node): + extraction = { + "nodes": [ + {"id": "src_app_main", "label": "main", "file_type": "code", + "source_file": "src/app.py", "source_location": "L1", "_origin": "ast"}, + doc_node, + ], + "edges": [{"source": "docs_guide_deploy", "target": "src_app_main", + "relation": "references", "source_file": "docs/guide.md", + "confidence": "EXTRACTED", **({"_origin": "semantic"} if "_origin" in doc_node else {})}], + "hyperedges": [], + } + G = build_from_json(extraction) + p = tmp_path / f"graph_{'stamped' if '_origin' in doc_node else 'bare'}.json" + to_json(G, {0: list(G.nodes)}, str(p)) + return p + + re_extraction = { + "nodes": [{"id": "src_app_main", "label": "main", "file_type": "code", + "source_file": "src/app.py", "source_location": "L2", "_origin": "ast"}], + "edges": [], "hyperedges": [], + } + + stamped = build_merge([re_extraction], graph_with(_line_located_doc_node(_origin="semantic"))) + assert "docs_guide_deploy" in stamped.nodes, "the stamped document node must survive" diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 190827d9ac..523beddb5f 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 4996beb787..80f8746aae 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -361,6 +361,18 @@ from pathlib import Path ast = json.loads(Path('.graphify_ast.json').read_text()) sem = json.loads(Path('.graphify_semantic.json').read_text()) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 190827d9ac..523beddb5f 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index abd2811d23..d4d86f3ed1 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index af3f723c78..8a4cce219c 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index abd2811d23..d4d86f3ed1 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index f9be846cbf..0cd4705ebf 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -425,6 +425,18 @@ from graphify.semantic_cleanup import sanitize_semantic_fragment ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index fd148d485d..ffc757e631 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -363,6 +363,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 3e70b050a4..e0782dd69d 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index abd2811d23..d4d86f3ed1 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 91ced60675..64c62f3de0 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -358,6 +358,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index abd2811d23..d4d86f3ed1 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 050667bc20..9279b32c2e 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -364,6 +364,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 20c7c0835c..d91fa12872 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -362,6 +362,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index b09ecca3c4..5eb98aba3a 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -393,6 +393,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding="utf-8")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding="utf-8")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index abd2811d23..d4d86f3ed1 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -366,6 +366,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 4996beb787..80f8746aae 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -361,6 +361,18 @@ from pathlib import Path ast = json.loads(Path('.graphify_ast.json').read_text()) sem = json.loads(Path('.graphify_semantic.json').read_text()) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index c527a12563..db284fed8b 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -301,6 +301,18 @@ from pathlib import Path ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index f9be846cbf..0cd4705ebf 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -425,6 +425,18 @@ from graphify.semantic_cleanup import sanitize_semantic_fragment ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) +# Stamp tier provenance on the semantic side. build._is_ast_tier reads +# _origin when present and otherwise guesses from the SHAPE of +# source_location ('L' means AST). A subagent that writes 'L12' for a +# doc section therefore made that node read as AST, and the next +# `graphify update` deleted it along with the re-extracted code file it +# seemed to belong to (#2843). extract() stamps its own items 'ast'; the +# semantic side must say so explicitly. +for n in sem['nodes']: + n.setdefault('_origin', 'semantic') +for e in sem['edges']: + e.setdefault('_origin', 'semantic') + # Merge: AST nodes first, semantic nodes deduplicated by id seen = {n['id'] for n in ast['nodes']} merged_nodes = list(ast['nodes']) diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 09e19ede00..ff3e1650bb 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -1142,6 +1142,35 @@ def _is_community_label_export_fix_line(line: str) -> bool: ) +def _is_semantic_origin_stamp_fix_line(line: str) -> bool: + """Whether a line is part of the Part-C `_origin` stamping fix (#2843). + + build._is_ast_tier reads ``_origin`` when present and otherwise guesses the + tier from the shape of ``source_location`` (``L`` means AST). Part C + never stamped the semantic side, so a subagent writing ``L12`` for a doc + section made that node read as AST and the next ``graphify update`` deleted + it with the re-extracted code file it seemed to belong to. Part C now + ``setdefault``s ``_origin: 'semantic'`` on every semantic node and edge + before the merge. These are the comment block and the four code lines. + """ + stripped = line.strip() + return ( + stripped in ( + "for n in sem['nodes']:", + "n.setdefault('_origin', 'semantic')", + "for e in sem['edges']:", + "e.setdefault('_origin', 'semantic')", + ) + or "Stamp tier provenance on the semantic side" in line + or "_origin when present and otherwise guesses from the SHAPE of" in line + or "source_location ('L' means AST). A subagent that writes 'L12' for a" in line + or "doc section therefore made that node read as AST, and the next" in line + or "`graphify update` deleted it along with the re-extracted code file it" in line + or "seemed to belong to (#2843). extract() stamps its own items 'ast'; the" in line + or "semantic side must say so explicitly." in line + ) + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -1163,6 +1192,7 @@ def _is_community_label_export_fix_line(line: str) -> bool: _is_uv_from_interpreter_fix_line, _is_semantic_cache_scope_fix_line, _is_community_label_export_fix_line, + _is_semantic_origin_stamp_fix_line, )