diff --git a/README.md b/README.md index 0c14d207c..0f91256ee 100644 --- a/README.md +++ b/README.md @@ -345,7 +345,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Common Lisp | `.lisp .cl .lsp .asd` (requires `uv tool install graphifyy[commonlisp]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | | Package manifests | `apm.yml` `pyproject.toml` `go.mod` `pom.xml` — one canonical package node per package (by name) plus `depends_on` edges, so a package referenced from many manifests is a single hub | -| Docs | `.md .mdx .qmd .html .txt .rst .yaml .yml` (markdown `[text](./other.md)` links and `[[wikilinks]]` become `references` edges between docs) | +| Docs | `.md .mdx .qmd .adoc .asciidoc .html .txt .rst .yaml .yml` (markdown `[text](./other.md)` links and `[[wikilinks]]`, and AsciiDoc `include::`/`xref:`/`link:` targets, become `references` edges between docs; AsciiDoc section titles become heading nodes like markdown headings) | | Office | `.docx .xlsx` (requires `uv tool install graphifyy[office]`) | | Google Workspace | `.gdoc .gsheet .gslides` (opt-in; requires `gws` auth and `--google-workspace`; Sheets need `uv tool install graphifyy[google]`) | | PDFs | `.pdf` | diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index 3c8a34b41..3e3783e4e 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -187,7 +187,7 @@ def normalize_node(raw: dict, index: int) -> dict: file_type = first_present(node, "file_type", "content_type", "artifact_type", default="") if not file_type: suffix = Path(str(source_file)).suffix.lower() - file_type = "document" if suffix in {".md", ".mdx", ".rst", ".txt"} else "code" + file_type = "document" if suffix in {".md", ".mdx", ".rst", ".txt", ".adoc", ".asciidoc"} else "code" node["id"] = str(node_id) node["label"] = str(label) diff --git a/graphify/cli.py b/graphify/cli.py index bd2956815..8c5824c7f 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -71,7 +71,7 @@ _HOOK_SOURCE_EXTS = ( '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', - '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', + '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', '.adoc', '.asciidoc', ) _GEMINI_NUDGE_TEXT = ( 'graphify: knowledge graph at graphify-out/. For focused questions, run ' diff --git a/graphify/detect.py b/graphify/detect.py index 3668eb6fc..ad2270b71 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -42,7 +42,7 @@ class FileType(str, Enum): _MTIME_SUBSECOND_S = 0.05 CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd'} -DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} +DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.adoc', '.asciidoc', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} diff --git a/graphify/extract.py b/graphify/extract.py index f68757a85..c3649657d 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -33,6 +33,7 @@ _read_text, ) from graphify.extractors.apex import extract_apex # noqa: F401 +from graphify.extractors.asciidoc import extract_asciidoc # noqa: F401 from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 from graphify.extractors.csharp import ( @@ -5228,6 +5229,8 @@ def add_existing_edge(edge: dict) -> None: ".sql": extract_sql, ".md": extract_markdown, ".mdx": extract_markdown, + ".adoc": extract_asciidoc, + ".asciidoc": extract_asciidoc, ".qmd": extract_markdown, ".skill": extract_markdown, ".pas": extract_pascal, diff --git a/graphify/extractors/asciidoc.py b/graphify/extractors/asciidoc.py new file mode 100644 index 000000000..6606d1f24 --- /dev/null +++ b/graphify/extractors/asciidoc.py @@ -0,0 +1,214 @@ +"""AsciiDoc extractor (#2938). + +``.adoc`` was in no extension set, so a project whose documentation is +AsciiDoc had none of it in the graph — only the PNGs beside it. This mirrors +:mod:`graphify.extractors.markdown`: the file is a ``page`` node, each +section title a ``heading`` node nested by level, and every local document +the file pulls in or points at becomes a ``references`` edge whose target +id is minted from the resolved path so it merges into that document's own +node. The semantic pass then digests the prose exactly as it does markdown. + +Links covered: ``include::other.adoc[]``, ``xref:other.adoc[text]`` / +``xref:other#anchor[]`` (an extension-less xref names a sibling ``.adoc``), +``link:guide.adoc[text]`` (a local path — ``link:https://...`` is external and +skipped), and the cross-document form of ``<>``. +Targets that still carry an unresolved attribute (``{docdir}/x.adoc``) are +skipped rather than guessed. + +Delimited blocks (listing ``----``, literal ``....``, comment ``////``, +passthrough ``++++``, example ``====``, sidebar ``****``, quote ``____``) +are skipped so their contents are neither headings nor links. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id +from graphify.security import sanitize_metadata + +ASCIIDOC_EXTENSIONS: frozenset[str] = frozenset({".adoc", ".asciidoc"}) + +# Documents an AsciiDoc file may reference; kept in step with markdown's set. +_LINKABLE_EXTS: frozenset[str] = frozenset({ + ".adoc", ".asciidoc", ".md", ".mdx", ".qmd", ".markdown", ".rst", ".txt", +}) + +_TITLE_RE = re.compile(r"^(=+)\s+(\S.*?)\s*=*\s*$") +_INCLUDE_RE = re.compile(r"^include::([^\[\]]+)\[[^\]]*\]") +_XREF_RE = re.compile(r"xref:([^\[\]\s]+)\[[^\]]*\]") +_LINK_RE = re.compile(r"link:([^\[\]\s]+)\[[^\]]*\]") +_ANGLE_XREF_RE = re.compile(r"<<([^>,\s]+)(?:,[^>]*)?>>") +_ATTR_LINE_RE = re.compile(r"^:([A-Za-z0-9_][A-Za-z0-9_-]*):\s*(.*)$") + +# Delimiter lines that open/close a block whose body must be skipped. A block +# closes on the SAME delimiter that opened it - same character AND same +# length, which is how AsciiDoc nests an example block inside an example +# block (`====` ... `======` ... `======` ... `====`). +_BLOCK_DELIM_CHARS = frozenset("-./+=*_") + + +def _block_delim(stripped: str) -> str | None: + if len(stripped) >= 4 and stripped[0] in _BLOCK_DELIM_CHARS and stripped == stripped[0] * len(stripped): + return stripped + return None + + +def _resolve_doc_link(raw: str, source_dir: Path) -> Path | None: + """Resolve a local document target to an absolute (normalised) path, or + None for external URLs, in-page anchors, attribute-bearing paths, and + non-document targets.""" + target = raw.strip() + if not target: + return None + target = target.split("#", 1)[0].split("?", 1)[0].strip() + if not target: + return None + low = target.lower() + if "://" in target or low.startswith(("mailto:", "tel:", "//", "data:")): + return None + if "{" in target or "}" in target: + return None # an unresolved AsciiDoc attribute; do not guess + suffix = Path(target).suffix.lower() + if suffix == "": + target += ".adoc" + suffix = ".adoc" + if suffix not in _LINKABLE_EXTS: + return None + candidate = Path(target) + if not candidate.is_absolute(): + candidate = source_dir / candidate + return Path(os.path.normpath(str(candidate))) + + +def extract_asciidoc(path: Path) -> dict: + """Extract page, section and document-reference structure from AsciiDoc.""" + try: + source = path.read_text(encoding="utf-8", errors="replace") + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + + def add_node(nid: str, label: str, line: int, node_kind: str = "heading", + extra: dict | None = None) -> None: + if nid in seen_ids: + return + seen_ids.add(nid) + node = {"id": nid, "label": label, "file_type": "document", + "node_kind": node_kind, "source_file": str_path, + "source_location": f"L{line}"} + if extra: + node.update(extra) + nodes.append(node) + + def add_edge(src: str, tgt: str, relation: str, line: int, + target_file: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0} + if target_file is not None: + edge["target_file"] = target_file + edges.append(edge) + + file_nid = _make_id(str_path) + source_dir = path.parent + linked: set[str] = set() + + def add_link(raw: str, line: int) -> None: + resolved = _resolve_doc_link(raw, source_dir) + if resolved is None: + return + # Same recipe as the target's own file node (see markdown.py): the + # absolute path, canonicalised by extract()'s post-pass, so the edge + # merges into the real document node instead of spawning a ghost. + tgt_nid = _make_id(str(resolved)) + if tgt_nid == file_nid or tgt_nid in linked: + return + linked.add(tgt_nid) + target_file = None + try: + if resolved.is_file(): + target_file = str(resolved) + except OSError: + pass + add_edge(file_nid, tgt_nid, "references", line, target_file=target_file) + + lines = source.splitlines() + # Header attributes (`:author: ...`) between the document title and the + # first blank line play the role markdown frontmatter does. + attributes: dict[str, str] = {} + in_header = True + heading_stack: list[tuple[int, str]] = [] + block_delim: str | None = None + doc_title: str | None = None + + for idx, line_text in enumerate(lines): + line_num = idx + 1 + stripped = line_text.strip() + delim = _block_delim(stripped) + if block_delim is not None: + if delim == block_delim: + block_delim = None + continue + if delim is not None: + block_delim = delim + continue + if stripped.startswith("//"): + continue # line comment + + if in_header: + if not stripped and doc_title is not None: + in_header = False + attr = _ATTR_LINE_RE.match(stripped) if stripped else None + if attr: + attributes[attr.group(1)] = attr.group(2).strip() + continue + + m = _INCLUDE_RE.match(stripped) + if m: + add_link(m.group(1), line_num) + continue + for m in _XREF_RE.finditer(line_text): + add_link(m.group(1), line_num) + for m in _LINK_RE.finditer(line_text): + add_link(m.group(1), line_num) + for m in _ANGLE_XREF_RE.finditer(line_text): + ref = m.group(1) + if "#" in ref and Path(ref.split("#", 1)[0]).suffix: + add_link(ref, line_num) # cross-document form only + + title = _TITLE_RE.match(line_text) + if not title: + continue + level = len(title.group(1)) # `=` is the document title (level 1) + text = title.group(2).strip() + if level == 1 and doc_title is None: + doc_title = text + continue # the page node carries the document title + h_nid = _make_id(stem, text) + if h_nid in seen_ids: + h_nid = _make_id(stem, text, str(line_num)) + add_node(h_nid, text, line_num) + while heading_stack and heading_stack[-1][0] >= level: + heading_stack.pop() + parent = heading_stack[-1][1] if heading_stack else file_nid + add_edge(parent, h_nid, "contains", line_num) + heading_stack.append((level, h_nid)) + + extra: dict = {} + if doc_title: + extra["title"] = doc_title + if attributes: + extra["frontmatter"] = sanitize_metadata(attributes) + # The page node goes first, as in markdown, regardless of when the title + # was seen. + nodes.insert(0, {"id": file_nid, "label": path.name, "file_type": "document", + "node_kind": "page", "source_file": str_path, + "source_location": "L1", **extra}) + return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0} diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index 50c1754a7..0be3c43d8 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -16,7 +16,7 @@ _MD_WIKILINK_RE = re.compile(r'(?>. +Also <> and xref:runbook#steps[again]. + +include::parts/alerts.adoc[] +include::{snippets}/gen.adoc[] + +---- +== not a heading inside a listing +xref:ignored.adoc[] +---- + +.... +include::ignored-too.adoc[] +.... + +=== Rollback + +== Monitor +""" + + +@pytest.fixture +def docs(tmp_path): + d = tmp_path / "docs" + (d / "parts").mkdir(parents=True) + (d / "guide.adoc").write_text(GUIDE, encoding="utf-8") + (d / "runbook.adoc").write_text("= Runbook\n\n== Steps\n", encoding="utf-8") + (d / "checklist.md").write_text("# Checklist\n\nSee [guide](./guide.adoc).\n", encoding="utf-8") + (d / "parts" / "alerts.adoc").write_text("== Alerts\n", encoding="utf-8") + return d + + +def _by(result): + nodes = {n["id"]: n for n in result["nodes"]} + edges = [(nodes[e["source"]]["label"] if e["source"] in nodes else e["source"], + e["relation"], + Path(e["target_file"]).name if e.get("target_file") else + (nodes[e["target"]]["label"] if e["target"] in nodes else e["target"])) + for e in result["edges"]] + return nodes, edges + + +# --------------------------------------------------------------------------- +# Detection, dispatch, slicing +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("ext", sorted(ASCIIDOC_EXTENSIONS)) +def test_asciidoc_is_a_document_everywhere_a_document_is_decided(ext): + assert ext in DOC_EXTENSIONS + assert classify_file(Path(f"x{ext}")) is FileType.DOCUMENT + assert _get_extractor(Path(f"x{ext}")) is extract_asciidoc + assert ext in _SPLITTABLE_TEXT_SUFFIXES # an oversized manual is sliced, not truncated + assert ext in _MD_LINKABLE_EXTS # a markdown doc may link to it + + +def test_detect_lists_adoc_files_as_documents(docs): + with redirect_stdout(io.StringIO()): + found = detect(docs.parent)["files"] + names = {Path(p).name for p in found["document"]} + assert {"guide.adoc", "runbook.adoc", "alerts.adoc", "checklist.md"} <= names + + +# --------------------------------------------------------------------------- +# Structure +# --------------------------------------------------------------------------- + +def test_the_page_node_carries_the_document_title_and_header_attributes(docs): + nodes, _ = _by(extract_asciidoc(docs / "guide.adoc")) + page = next(n for n in nodes.values() if n["node_kind"] == "page") + assert page["label"] == "guide.adoc" + assert page["title"] == "Operations Guide" + assert page["frontmatter"]["author"] == "Ops Team" + assert page["file_type"] == "document" + + +def test_sections_nest_by_level_like_markdown_headings(docs): + nodes, edges = _by(extract_asciidoc(docs / "guide.adoc")) + headings = [n["label"] for n in nodes.values() if n["node_kind"] == "heading"] + assert headings == ["Deploy", "Rollback", "Monitor"] + assert ("guide.adoc", "contains", "Deploy") in edges + assert ("Deploy", "contains", "Rollback") in edges + assert ("guide.adoc", "contains", "Monitor") in edges + + +def test_delimited_blocks_and_comments_are_skipped(docs): + nodes, edges = _by(extract_asciidoc(docs / "guide.adoc")) + labels = {n["label"] for n in nodes.values()} + assert "not a heading inside a listing" not in labels + assert not any(t in ("ignored.adoc", "ignored-too.adoc") for _, _, t in edges) + + +def test_a_repeated_section_title_still_yields_two_nodes(tmp_path): + p = tmp_path / "a.adoc" + p.write_text("== Usage\n\n== Usage\n", encoding="utf-8") + nodes, _ = _by(extract_asciidoc(p)) + assert sum(1 for n in nodes.values() if n["label"] == "Usage") == 2 + + +def test_a_document_without_a_title_line_still_has_a_page_node(tmp_path): + p = tmp_path / "frag.adoc" + p.write_text("Just prose.\n\n== Part\n", encoding="utf-8") + nodes, _ = _by(extract_asciidoc(p)) + page = next(n for n in nodes.values() if n["node_kind"] == "page") + assert "title" not in page and page["label"] == "frag.adoc" + + +def test_an_unreadable_file_reports_an_error_not_a_crash(tmp_path): + r = extract_asciidoc(tmp_path / "missing.adoc") + assert r["nodes"] == [] and "error" in r + + +# --------------------------------------------------------------------------- +# References +# --------------------------------------------------------------------------- + +def test_include_xref_link_and_angle_xref_become_references(docs): + _, edges = _by(extract_asciidoc(docs / "guide.adoc")) + refs = {t for s, r, t in edges if r == "references"} + assert refs == {"runbook.adoc", "alerts.adoc", "checklist.md"} + + +def test_each_target_is_referenced_once_however_often_it_is_named(docs): + """runbook is named by xref twice (one extension-less, with an anchor).""" + _, edges = _by(extract_asciidoc(docs / "guide.adoc")) + assert sum(1 for s, r, t in edges if r == "references" and t == "runbook.adoc") == 1 + + +@pytest.mark.parametrize("raw", [ + "https://example.com/x.adoc", "mailto:a@b.c", "#anchor", "", "{docdir}/x.adoc", + "diagram.png", "script.py", +]) +def test_external_anchor_attribute_and_non_document_targets_are_skipped(raw, tmp_path): + assert _resolve_doc_link(raw, tmp_path) is None + + +def test_an_extensionless_xref_names_a_sibling_adoc(tmp_path): + assert _resolve_doc_link("runbook#steps", tmp_path) == tmp_path / "runbook.adoc" + assert _resolve_doc_link("../guide.adoc", tmp_path / "parts") == tmp_path / "guide.adoc" + + +def test_an_existing_target_is_stamped_for_the_incremental_remap(docs): + r = extract_asciidoc(docs / "guide.adoc") + stamped = {Path(e["target_file"]).name for e in r["edges"] if e.get("target_file")} + assert {"runbook.adoc", "alerts.adoc", "checklist.md"} <= stamped + + +# --------------------------------------------------------------------------- +# Corpus level +# --------------------------------------------------------------------------- + +def test_references_merge_into_the_linked_documents_own_nodes(docs): + root = docs.parent + files = sorted(p for p in docs.rglob("*") if p.is_file()) + with redirect_stdout(io.StringIO()): + g = extract(files, cache_root=root, root=root) + labels = {n["id"]: n["label"] for n in g["nodes"]} + refs = {(labels[e["source"]], labels.get(e["target"], "DANGLING")) + for e in g["edges"] if e["relation"] == "references"} + assert ("guide.adoc", "runbook.adoc") in refs + assert ("guide.adoc", "alerts.adoc") in refs + assert ("guide.adoc", "checklist.md") in refs + assert ("checklist.md", "guide.adoc") in refs # markdown -> asciidoc link resolves too + assert "DANGLING" not in {t for _, t in refs} + assert not any("target_file" in e for e in g["edges"]) + + +def test_a_nested_block_closes_only_on_its_own_delimiter_length(tmp_path): + """`====` opens an example block; a `======` inside it is a nested block, + not the close - the outer block ends at the next `====`.""" + p = tmp_path / "n.adoc" + p.write_text("== Real" + chr(10) + "====" + chr(10) + "== Inside outer" + chr(10) + "======" + chr(10) + + "== Inside nested" + chr(10) + "======" + chr(10) + "== Still inside outer" + chr(10) + "====" + chr(10) + + "== After" + chr(10), encoding="utf-8") + nodes, _ = _by(extract_asciidoc(p)) + assert [n["label"] for n in nodes.values() if n["node_kind"] == "heading"] == ["Real", "After"] + + +def test_both_extensions_are_in_the_hook_source_list(): + from graphify.cli import _HOOK_SOURCE_EXTS + assert ".adoc" in _HOOK_SOURCE_EXTS and ".asciidoc" in _HOOK_SOURCE_EXTS