From 97e938e00cc4108bb87ca4ce3537921460386c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20S=C3=B6semann?= Date: Wed, 26 Aug 2026 17:25:45 +0200 Subject: [PATCH 1/2] feat(apex): parse Apex with tree-sitter, add calls edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Apex extractor never produced calls edges, and its docstring said why: no tree-sitter grammar on PyPI. That is out of date — aheber/tree-sitter-sfapex exists, is MIT, and uses the same node type and field names as tree-sitter-java for declarations and calls, so the shared engine walk drives it from a config. On my-org-butler, 55 Apex files: 592 calls edges where there were 0, across 225 distinct targets. Not the Java grammar: parsing those same files with tree-sitter-java yields 740 ERROR nodes in 53 of 55, silently. The sfapex grammar parses all 55 clean. The regex extractor stays as the fallback, like Pascal's, so Apex still extracts without the optional extra — minus calls. extends/implements, the SObject behind a trigger or a SOQL FROM, DML, and `new X()` are handled in the Apex extractor rather than by widening the Java-gated branches in the engine. Two filters keep collection built-ins from becoming god-nodes: `new List<>()` is not a dependency, and `rows.add(x)` must not bind to whichever class happens to declare add(). Shared core gains one optional LanguageConfig field for grammars only reachable through tree-sitter-language-pack, plus the loader branch using it. --- graphify/extractors/apex.py | 279 +++++++++++++++++++++++++++++++- graphify/extractors/engine.py | 21 ++- graphify/extractors/models.py | 5 + pyproject.toml | 10 +- tests/test_apex_calls.py | 290 ++++++++++++++++++++++++++++++++++ 5 files changed, 594 insertions(+), 11 deletions(-) create mode 100644 tests/test_apex_calls.py diff --git a/graphify/extractors/apex.py b/graphify/extractors/apex.py index 928923a640..8db23cecfe 100644 --- a/graphify/extractors/apex.py +++ b/graphify/extractors/apex.py @@ -1,14 +1,79 @@ -"""Apex extractor. Moved verbatim from graphify/extract.py.""" +"""Apex extractor: tree-sitter when the grammar is installed, regex otherwise.""" from __future__ import annotations from pathlib import Path + from graphify.extractors.base import _file_stem, _make_id +from graphify.extractors.models import LanguageConfig + +# The sfapex grammar (aheber/tree-sitter-sfapex) uses the same node type and field +# names as tree-sitter-java for declarations and calls, so the shared engine walk +# drives it unchanged. Apex has no import statements, hence no import_types. +_APEX_CONFIG = LanguageConfig( + ts_module="tree_sitter_language_pack", + ts_language_pack_name="apex", + class_types=frozenset({ + "class_declaration", "interface_declaration", "enum_declaration", + "trigger_declaration", + }), + function_types=frozenset({"method_declaration", "constructor_declaration"}), + call_types=frozenset({"method_invocation"}), + call_function_field="name", + function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), +) def extract_apex(path: Path) -> dict: + """Extract an Apex .cls or .trigger file. + + Prefers the real parser; falls back to the regex extractor when the grammar + is not installed, mirroring how Pascal treats its optional grammar. The + fallback keeps every Apex corpus working without the extra, at lower + fidelity and with no `calls` edges. + """ + ast = _extract_apex_ast(path) + return ast if ast is not None else _extract_apex_regex(path) + + +def _extract_apex_ast(path: Path) -> dict | None: + """Engine walk plus the Apex-only constructs the generic walk cannot know. + + Returns None when the grammar is unavailable or the file does not parse, so + the caller can fall back rather than emit a half-empty result. + """ + try: + from tree_sitter_language_pack import get_parser + except Exception: + return None + try: + parser = get_parser("apex") + source = path.read_bytes() + except Exception: + return None + + from graphify.extractors.engine import _extract_generic + + result = _extract_generic(path, _APEX_CONFIG) + if result.get("error"): + return None + try: + root = parser.parse(source).root_node + except Exception: + return None + _add_apex_specifics(path, root, source, result) + raw_calls = result.get("raw_calls") + if raw_calls: + result["raw_calls"] = [ + rc for rc in raw_calls + if str(rc.get("callee", "")).lower() not in _APEX_BUILTIN_METHODS + ] + return result + + +def _extract_apex_regex(path: Path) -> dict: """Extract classes, interfaces, enums, methods, and Salesforce constructs from - Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI).""" + Apex .cls and .trigger files using regex. Fallback for a missing grammar.""" import re as _re try: source = path.read_text(encoding="utf-8", errors="replace") @@ -213,3 +278,213 @@ def add_edge(src: str, tgt: str, relation: str, line: int, add_edge(src, dml_nid, "uses", lineno, confidence="INFERRED") return {"nodes": nodes, "edges": edges} + + +# ── Apex constructs the generic engine walk has no concept of ───────────────── + +_DML_TYPES = frozenset({"insert", "update", "delete", "upsert", "merge", "undelete"}) + +# Annotations that make a method an entry point reachable from outside Apex. +_ENTRY_POINT_ANNOTATIONS = frozenset({"auraenabled", "invocablemethod"}) + +# Collection and primitive constructors. `new List()` appears in nearly +# every method, so treating it as a call site builds a god-node that collects an +# edge from the whole codebase and tells you nothing — the same reason +# base.py filters language built-ins. Platform types that do carry meaning +# (HttpRequest, and any custom type) are deliberately NOT filtered. +# Collection and Map/Set/String methods. These are the receiver's methods, not a +# user method, but the cross-file call resolver matches unresolved calls by bare +# name — so `rows.add(x)` in twenty classes all bind to a user class that happens +# to define `add`, inventing twenty dependencies. Same-file calls are resolved +# against real declarations before this applies, so a class calling its own +# `add()` keeps its edge. Deliberately conservative: `execute` and `send` are NOT +# here, because they are commonly real user methods. +# Note the asymmetry that makes this necessary: a name defined by MANY classes is +# already safe, because the resolver refuses to bind an ambiguous name. The +# damage comes from a name defined by exactly ONE class — `send` in one class +# collects every `new Http().send(req)` in the codebase. `execute` is left out +# for exactly that reason: it is declared by every invocable class, so it is +# ambiguous and only ever resolves within a file. +_APEX_BUILTIN_METHODS = frozenset({ + # Collections and Map/Set + "add", "addall", "get", "put", "putall", "size", "isempty", "clear", + "contains", "containskey", "keyset", "values", "remove", "indexof", "sort", + "deepclone", "clone", + # Http/HttpRequest/HttpResponse + "send", "getbody", "setbody", "getstatuscode", "setstatuscode", + "setheader", "getheader", "setendpoint", "setmethod", + # JSON, String, Object + "serialize", "deserialize", "deserializeuntyped", "escapesinglequotes", + "isblank", "isnotblank", "valueof", "tostring", "equals", "hashcode", +}) + +_APEX_BUILTIN_CONSTRUCTORS = frozenset({ + "list", "set", "map", "blob", "object", + "string", "integer", "long", "double", "decimal", "boolean", + "date", "datetime", "time", "id", +}) + + +def _apex_text(node, source: bytes) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + +def _add_apex_specifics(path: Path, root, source: bytes, result: dict) -> None: + """Add inheritance, SObject usage, DML and entry points to an engine result. + + The engine walk covers declarations and calls, which are shaped like Java. + Everything here is Apex-only: `extends`/`implements` (the engine's handling + is gated on the Java grammar), the SObject a trigger fires on, the SObject + behind a SOQL `FROM` or SOSL `RETURNING`, and DML statements. Node ids reuse + the engine's scheme so the two halves land on the same nodes. + """ + str_path = str(path) + stem = _file_stem(path) + file_nid = _make_id(str_path) + nodes: list[dict] = result["nodes"] + edges: list[dict] = result["edges"] + seen_ids: set[str] = {n["id"] for n in nodes if n.get("id")} + seen_edges = {(e.get("source"), e.get("target"), e.get("relation")) for e in edges} + + def add_stub(nid: str, label: str) -> None: + """Sourceless placeholder — see the note in _extract_apex_regex.""" + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": "", "source_location": ""}) + + def add_edge(src: str, tgt: str, relation: str, line: int) -> None: + key = (src, tgt, relation) + if src == tgt or key in seen_edges: + return + seen_edges.add(key) + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "INFERRED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + + def type_ref(name: str) -> str: + local = _make_id(stem, name) + if local in seen_ids: + return local + nid = _make_id(name) + add_stub(nid, name) + return nid + + def named(node, name: str): + return node.child_by_field_name(name) + + def enclosing_callable(node) -> str: + """Nearest enclosing method/constructor node id, else the type or file.""" + cur = node.parent + while cur is not None: + if cur.type in ("method_declaration", "constructor_declaration"): + name_node = named(cur, "name") + if name_node is not None: + nid = _make_id(enclosing_owner(cur), + _apex_text(name_node, source)) + if nid in seen_ids: + return nid + cur = cur.parent + return enclosing_owner(node) + + def enclosing_owner(node) -> str: + """Nearest enclosing type or trigger node id, else the file node.""" + cur = node.parent + while cur is not None: + if cur.type in ("class_declaration", "interface_declaration", + "enum_declaration", "trigger_declaration"): + name_node = named(cur, "name") + if name_node is not None: + nid = _make_id(stem, _apex_text(name_node, source)) + if nid in seen_ids: + return nid + cur = cur.parent + return file_nid + + stack = [root] + while stack: + node = stack.pop() + stack.extend(node.children) + t = node.type + line = node.start_point[0] + 1 + + if t in ("class_declaration", "interface_declaration"): + name_node = named(node, "name") + if name_node is None: + continue + owner = _make_id(stem, _apex_text(name_node, source)) + if owner not in seen_ids: + continue + for child in node.children: + if child.type == "superclass": + for sub in child.children: + if sub.type in ("type_identifier", "scoped_type_identifier"): + add_edge(owner, type_ref(_apex_text(sub, source)), + "extends", line) + elif child.type in ("interfaces", "extends_interfaces"): + for sub in child.named_children: + for entry in (sub.named_children if sub.type == "type_list" else [sub]): + if entry.type in ("type_identifier", "scoped_type_identifier", + "generic_type"): + raw = _apex_text(entry, source).split("<", 1)[0].strip() + relation = ("extends" if t == "interface_declaration" + else "implements") + add_edge(owner, type_ref(raw), relation, line) + + elif t == "trigger_declaration": + name_node, obj_node = named(node, "name"), named(node, "object") + if name_node is not None and obj_node is not None: + trig = _make_id(stem, _apex_text(name_node, source)) + if trig in seen_ids: + add_edge(trig, type_ref(_apex_text(obj_node, source)), "uses", line) + + elif t == "storage_identifier" and node.parent is not None \ + and node.parent.type == "from_clause": + add_edge(enclosing_owner(node), + type_ref(_apex_text(node, source).split(".")[0]), "uses", line) + + elif t == "sobject_return": + for sub in node.children: + if sub.type == "identifier": + add_edge(enclosing_owner(node), + type_ref(_apex_text(sub, source)), "uses", line) + break + + elif t == "object_creation_expression": + # `new Other()` is a call site whose callee sits in the `type` field, + # so the shared call walk (which reads `name`) never sees it. + type_node = named(node, "type") + if type_node is not None: + raw = _apex_text(type_node, source).split("<", 1)[0].strip() + if (raw and raw[:1].isalpha() + and raw.lower() not in _APEX_BUILTIN_CONSTRUCTORS): + add_edge(enclosing_callable(node), type_ref(raw), "calls", line) + + elif t == "dml_type": + op = _apex_text(node, source).strip().lower() + if op in _DML_TYPES: + dml_nid = _make_id(f"dml_{op}") + if dml_nid not in seen_ids: + seen_ids.add(dml_nid) + nodes.append({"id": dml_nid, "label": op, "file_type": "code", + "source_file": str_path, + "source_location": f"L{line}"}) + add_edge(enclosing_owner(node), dml_nid, "uses", line) + + elif t in ("method_declaration", "constructor_declaration"): + name_node = named(node, "name") + if name_node is None: + continue + owner = enclosing_owner(node) + method_nid = _make_id(owner, _apex_text(name_node, source)) + if method_nid not in seen_ids: + continue + for mods in node.children: + if mods.type != "modifiers": + continue + for anno in mods.children: + if anno.type != "annotation": + continue + raw = _apex_text(anno, source).lstrip("@").split("(")[0].strip().lower() + if raw in _ENTRY_POINT_ANNOTATIONS: + add_edge(file_nid, method_nid, "contains", line) diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index e883747784..690e175811 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -2876,15 +2876,20 @@ def _extract_generic( mask the wrapper and parse just the embedded ``