-
-
Notifications
You must be signed in to change notification settings - Fork 10.8k
feat(config): let a project declare what an ambiguous extension means via .graphifyrc (#2961) #3075
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: v8
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -921,11 +921,19 @@ def cache_dir(root: Path = Path("."), kind: str = "ast", | |
| return d | ||
|
|
||
|
|
||
| def _salted_key(h: str, salt: str | None) -> str: | ||
| """Derive the on-disk key from a content hash plus optional salt.""" | ||
| if not salt: | ||
| return h | ||
| return hashlib.sha256(f"{h}:{salt}".encode("utf-8")).hexdigest() | ||
|
|
||
|
|
||
| def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", | ||
| cache_root: Path | None = None, prompt: "str | Path | None" = None, | ||
| prompt_file: "str | Path | None" = None, | ||
| allow_legacy: bool = True, | ||
| allow_partial: bool = False) -> dict | None: | ||
| allow_partial: bool = False, | ||
| salt: str | None = None) -> dict | None: | ||
| """Return cached extraction for this file if hash matches, else None. | ||
|
|
||
| Cache key: SHA256 of file contents. | ||
|
|
@@ -954,6 +962,10 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", | |
| so :func:`check_semantic_cache` can report N to the user. Callers that must | ||
| not mix vintages within one entry (see :func:`save_semantic_cache`'s | ||
| ``merge_existing``) pass allow_legacy=False. | ||
| ``salt`` folds extra material into the key for an entry whose validity | ||
| depends on more than the file's bytes — a file the project remaps to | ||
| another language (#2961) parses to a different graph from the same | ||
| content, so it must not hit the entry produced under the old extractor. | ||
| Returns None if no cache entry or file has changed. | ||
| """ | ||
| global _legacy_semantic_hits, _corrupt_cache_entries | ||
|
|
@@ -962,6 +974,7 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", | |
| h = file_hash(path, root, cache_root=cache_root) | ||
| except OSError: | ||
| return None | ||
| h = _salted_key(h, salt) | ||
| prompt_fp = _resolve_prompt_fp(prompt, prompt_file) | ||
| entry = cache_dir(location, kind, prompt_fp) / f"{h}.json" | ||
| legacy_hit = False | ||
|
|
@@ -1029,9 +1042,13 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", | |
|
|
||
| def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "ast", | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
fans out to 7 callees (efferent coupling); 24 callers depend on it (afferent coupling). Grounded coupling-delta finding (deterministic), not an LLM guess. |
||
| cache_root: Path | None = None, prompt: "str | Path | None" = None, | ||
| prompt_file: "str | Path | None" = None) -> None: | ||
| prompt_file: "str | Path | None" = None, | ||
| salt: str | None = None) -> None: | ||
| """Save extraction result for this file. | ||
|
|
||
| ``salt`` must match what the corresponding :func:`load_cached` passes | ||
| (see there); it namespaces the key, not the directory. | ||
|
|
||
| Stores as graphify-out/cache/{kind}/{hash}.json where hash = SHA256 of current file contents. | ||
| result should be a dict with 'nodes' and 'edges' lists. | ||
|
|
||
|
|
@@ -1078,7 +1095,7 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a | |
| # the entry replays portably under any root (#2257). Strictly after the | ||
| # source_file pass, which owns that field's bare-relative format. | ||
| _relativize_ids_in(on_disk, p, root) | ||
| h = file_hash(p, root, cache_root=cache_root) | ||
| h = _salted_key(file_hash(p, root, cache_root=cache_root), salt) | ||
| location = cache_root if cache_root is not None else root | ||
| target_dir = cache_dir(location, kind, _resolve_prompt_fp(prompt, prompt_file)) | ||
| entry = target_dir / f"{h}.json" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,7 @@ | |
| google_workspace_enabled, | ||
| ) | ||
| from graphify.paths import GRAPHIFY_OUT, out_path | ||
| from graphify.rcfile import activate_language_overrides, effective_suffix | ||
|
|
||
|
|
||
| class FileType(str, Enum): | ||
|
|
@@ -512,7 +513,9 @@ def classify_file(path: Path) -> FileType | None: | |
| # Compound extensions must be checked before simple suffix lookup | ||
| if path.name.lower().endswith(".blade.php"): | ||
| return FileType.CODE | ||
| ext = path.suffix.lower() | ||
| # A project may declare what an ambiguous extension means to it | ||
| # (.graphifyrc `language.inc=php`, #2961); classify by the declared one. | ||
| ext = effective_suffix(path).lower() | ||
| if not ext: | ||
| return _shebang_file_type(path) | ||
| if ext in CODE_EXTENSIONS: | ||
|
|
@@ -1514,6 +1517,9 @@ def _resolves_under_root(path: Path, root: Path) -> bool: | |
|
|
||
| def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
fans out to 16 callees (efferent coupling); 110 callers depend on it (afferent coupling). Grounded coupling-delta finding (deterministic), not an LLM guess. |
||
| root = root.resolve() | ||
| # The project's extension->language declarations (#2961) shape | ||
| # classification for this scan. | ||
| activate_language_overrides(root) | ||
| configured_out_dir = root / GRAPHIFY_OUT | ||
| configured_out_names = {configured_out_dir.name} | ||
| try: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,6 +26,13 @@ | |
| from .pascal_resolution import resolve_pascal_inherited_calls | ||
|
|
||
| # --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) --- | ||
| from graphify.rcfile import ( | ||
| activate_language_overrides, | ||
| cache_salt, | ||
| effective_suffix, | ||
| get_language_overrides, | ||
| set_language_overrides, | ||
| ) | ||
| from graphify.extractors.base import ( # noqa: F401 | ||
| _LANGUAGE_BUILTIN_GLOBALS, | ||
| _file_stem, | ||
|
|
@@ -2193,7 +2200,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: | |
| """True when the file's language resolves identifiers case-insensitively (#1581).""" | ||
| if not source_file: | ||
| return False | ||
| return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS | ||
| return effective_suffix(str(source_file)).lower() in _CASE_INSENSITIVE_EXTS | ||
|
|
||
|
|
||
| # Language interop families for cross-file call resolution. A call in one language | ||
|
|
@@ -2239,7 +2246,7 @@ def _lang_family(source_file: object) -> str | None: | |
| """Interop family of the file's language, or None when unknown/not code.""" | ||
| if not source_file: | ||
| return None | ||
| return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower()) | ||
| return _LANG_FAMILY_BY_EXT.get(effective_suffix(str(source_file)).lower()) | ||
|
|
||
|
|
||
| # A language's own built-in throwable hierarchy, keyed by the interop family of | ||
|
|
@@ -5387,6 +5394,12 @@ def _get_extractor(path: Path) -> Any | None: | |
| # (#1377). apm.yml would otherwise be a .yml document handled by the LLM. | ||
| if is_package_manifest_path(path): | ||
| return extract_package_manifest | ||
| # A project-declared remap (.graphifyrc `language.inc=php`, #2961) wins | ||
| # over every sniff below: the user has said what the extension means in | ||
| # this repo, so a remapped `.h`/`.m` also skips the C++/ObjC probes. | ||
| remapped = effective_suffix(path) | ||
| if remapped != path.suffix: | ||
| return _DISPATCH.get(remapped) or _DISPATCH.get(remapped.lower()) | ||
| # `.h` is C/C++/ObjC-ambiguous; route Objective-C headers to extract_objc | ||
| # (the suffix map sends `.h` to extract_c, which can't read @interface etc.). | ||
| # ObjC sniffing has priority over the C++ sniff: an Objective-C++ header can | ||
|
|
@@ -5429,6 +5442,13 @@ def _safe_extract_with_xaml_root(extractor, path: Path, root: Path) -> dict: | |
| _XAML_ACTIVE_EXTRACT_ROOT = previous_root | ||
|
|
||
|
|
||
| def _worker_init(language_overrides: dict[str, str]) -> None: | ||
| """Pool initializer. Under ``spawn`` a worker re-imports the package and | ||
| starts with no overrides; hand it the parent's so `_get_extractor` and the | ||
| cache key agree across processes (#2961).""" | ||
| set_language_overrides(language_overrides) | ||
|
|
||
|
|
||
| def _extract_single_file(args: tuple) -> tuple[int, dict]: | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
fans out to 7 callees (efferent coupling). Grounded coupling-delta finding (deterministic), not an LLM guess. |
||
| """Worker function for parallel extraction. Runs in a subprocess. | ||
|
|
||
|
|
@@ -5457,7 +5477,7 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: | |
|
|
||
| # Check cache first (avoid re-extraction) | ||
| if not bypass_cache: | ||
| cached = load_cached(path, root, cache_root=cache_location) | ||
| cached = load_cached(path, root, cache_root=cache_location, salt=cache_salt(path)) | ||
| if cached is not None: | ||
| return idx, cached | ||
|
|
||
|
|
@@ -5472,7 +5492,7 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: | |
| # byte-stable across runs and silently blinds affected/explain to and | ||
| # through the file (#1666); skipping the write lets a rerun self-heal. | ||
| if not bypass_cache and "error" not in result and result.get("nodes"): | ||
| save_cached(path, result, root, cache_root=cache_location) | ||
| save_cached(path, result, root, cache_root=cache_location, salt=cache_salt(path)) | ||
| return idx, result | ||
|
|
||
|
|
||
|
|
@@ -5538,7 +5558,11 @@ def _extract_parallel( | |
| failed: list[int] = [] # positions into uncached_work whose future failed | ||
| _PROGRESS_INTERVAL = 100 | ||
| try: | ||
| with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool: | ||
| with concurrent.futures.ProcessPoolExecutor( | ||
| max_workers=max_workers, | ||
| initializer=_worker_init, | ||
| initargs=(get_language_overrides(),), | ||
| ) as pool: | ||
| futures = { | ||
| pool.submit(_extract_single_file, item): pos | ||
| for pos, item in enumerate(work_items) | ||
|
|
@@ -5637,7 +5661,7 @@ def _extract_sequential( | |
| result = _safe_extract_with_xaml_root(extractor, path, root) | ||
| # See _extract_single_file: don't cache an anomalous zero-node result (#1666). | ||
| if not bypass_cache and "error" not in result and result.get("nodes"): | ||
| save_cached(path, result, root, cache_root=cache_location) | ||
| save_cached(path, result, root, cache_root=cache_location, salt=cache_salt(path)) | ||
| per_file[idx] = result | ||
| if total_files >= _PROGRESS_INTERVAL: | ||
| # Consistent denominator with the intermediate lines (#1693). | ||
|
|
@@ -5742,6 +5766,17 @@ def extract( | |
| root = cache_root | ||
| root = root.resolve() | ||
|
|
||
| # Project-level extension->language declarations (#2961). detect() has | ||
| # usually activated them for this root already; a direct library caller | ||
| # (the skill runbook, the MCP server, the issue's own repro) gets them here. | ||
| _overrides = activate_language_overrides(root) | ||
| if _overrides: | ||
| print( | ||
| " language overrides (.graphifyrc): " | ||
| + ", ".join(f"{k} -> {v}" for k, v in sorted(_overrides.items())), | ||
| flush=True, | ||
| ) | ||
|
|
||
| # #1774: the cache is an OUTPUT, so when no explicit cache_root is given it is | ||
| # written under the current working directory — never `root` (the inferred | ||
| # common parent of the inputs), which would drop graphify-out/ inside a | ||
|
|
@@ -5761,7 +5796,7 @@ def extract( | |
| continue | ||
| bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES | ||
| if not bypass_cache: | ||
| cached = load_cached(path, root, cache_root=cache_location) | ||
| cached = load_cached(path, root, cache_root=cache_location, salt=cache_salt(path)) | ||
| if cached is not None: | ||
| per_file[i] = cached | ||
| continue | ||
|
|
@@ -6396,7 +6431,7 @@ def _learn(e: dict) -> None: | |
| _php_exts = {".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps"} | ||
| _php_sel = [ | ||
| (r, p) for r, p in zip(per_file, paths) | ||
| if p.suffix.lower() in _php_exts and not p.name.lower().endswith(".blade.php") | ||
| if effective_suffix(p).lower() in _php_exts and not p.name.lower().endswith(".blade.php") | ||
| ] | ||
| if _php_sel: | ||
| try: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
load_cached()fans out to 8 callees (efferent coupling); 48 callers depend on it (afferent coupling).
Grounded coupling-delta finding (deterministic), not an LLM guess.