diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad98130..c5c3bbc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- Improve Markdown as first-class agent context. Repeated automatic heading + slugs now follow deterministic source-order suffixes, every structural block + has a section-qualified identity, and project resolution connects exact, + extensionless, repository-root, directory-index, and unique-wikilink targets + to document headings or source-file inventory. Missing fragments and + ambiguous targets remain unresolved, while exact document relationships now + survive the default low inference level. Advance extraction semantics to v12 + so affected cached facts rebuild. + - Improve natural code discovery on independently reviewed Python and TypeScript libraries. Ranking now recognizes common operation vocabulary, compound protocol owners, container roles, and decoder-chain intent without diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index dfba4c6b..3fae291d 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -388,6 +388,14 @@ changing node identity; consumers must preserve unknown attributes, edge direction, multiplicity, and source ranges. Markdown frontmatter is part of the file hash, so metadata-only edits invalidate compatible extraction/cache entries and are rebuilt under the current extraction semantics version. +Structural blocks now carry section-qualified names and automatic duplicate +heading slugs use deterministic source-order suffixes. Project resolution may +connect local document links to a unique heading, document root, directory +index, or source-file inventory node. Extension inference and wikilink stem +matching are bounded closed rules; ambiguity or a missing fragment never picks +one candidate or falls back to the document root. These additive attributes and +relationships rebuild under extraction semantics v12; graph schema v1 and +existing relationship direction remain unchanged. HTML (`.html`/`.htm`) now has the same source-driven structural contract. HTML nodes and link evidence preserve exact source ranges and deterministic order; diff --git a/crates/compass-core/src/pipeline.rs b/crates/compass-core/src/pipeline.rs index 914c541c..d27e792a 100644 --- a/crates/compass-core/src/pipeline.rs +++ b/crates/compass-core/src/pipeline.rs @@ -8231,14 +8231,18 @@ mod tests { let directory = tempfile::tempdir()?; fs::write( directory.path().join("guide.md"), - "# Guide\n[Implementation](documented.rs)\n", + "# Guide\n[Implementation](documented.rs)\n[Details](details#usage)\n", + )?; + fs::write( + directory.path().join("details.md"), + "# Details\n\n## Usage\n\nExact project documentation.\n", )?; fs::write( directory.path().join("documented.rs"), "pub fn documented() {}\n", )?; let mut options = BuildOptions::new(directory.path()); - options.inference_level = InferenceLevel::Max; + options.inference_level = InferenceLevel::Low; options.no_cluster = true; options.no_viz = true; options.force = true; @@ -8251,6 +8255,16 @@ mod tests { .any(|edge| edge.kind.as_str() == "documents"), "graph={graph:#?}" ); + assert!( + graph.links.iter().any(|edge| { + edge.kind.as_str() == "references" + && edge + .relationship_site + .as_ref() + .is_some_and(|site| site.file == "guide.md") + }), + "exact document resolution must survive default-low publication: {graph:#?}" + ); Ok(()) } diff --git a/crates/compass-languages/src/lib.rs b/crates/compass-languages/src/lib.rs index 4f1f400d..d0190fe9 100644 --- a/crates/compass-languages/src/lib.rs +++ b/crates/compass-languages/src/lib.rs @@ -18,7 +18,7 @@ mod fortran; pub mod frameworks; /// Version of the extraction contract consumed by graph publication. -pub const EXTRACTION_SEMANTICS_VERSION: &str = "compass.languages.extraction/11"; +pub const EXTRACTION_SEMANTICS_VERSION: &str = "compass.languages.extraction/12"; mod go; mod groovy; mod html; diff --git a/crates/compass-languages/src/markdown.rs b/crates/compass-languages/src/markdown.rs index 826c94bb..f5d99395 100644 --- a/crates/compass-languages/src/markdown.rs +++ b/crates/compass-languages/src/markdown.rs @@ -63,6 +63,7 @@ pub(crate) fn extract_source( seen_nodes: HashSet::new(), heading_stack: Vec::new(), heading_occurrences: HashMap::new(), + heading_slug_occurrences: HashMap::new(), heading_targets: HashMap::new(), reference_definitions: HashMap::new(), footnote_definitions: HashMap::new(), @@ -167,6 +168,7 @@ struct State<'source, 'path> { seen_nodes: HashSet, heading_stack: Vec, heading_occurrences: HashMap, + heading_slug_occurrences: HashMap, heading_targets: HashMap>, reference_definitions: HashMap, footnote_definitions: HashMap>, @@ -203,6 +205,12 @@ struct LinkSite { line_start: usize, } +struct DocumentTargetHint { + path: String, + extension_inferred: bool, + root_relative: bool, +} + impl State<'_, '_> { fn add_root(&mut self, id: String, metadata: Option>) { self.seen_nodes.insert(id.clone()); @@ -328,10 +336,8 @@ impl State<'_, '_> { .to_owned(), ), ); - extra.insert( - "anchor_slug".to_owned(), - Value::String(slugify(explicit_id.as_deref().unwrap_or(title))), - ); + let anchor_slug = self.heading_anchor_slug(title, explicit_id.as_deref()); + extra.insert("anchor_slug".to_owned(), Value::String(anchor_slug.clone())); if let Some(explicit_id) = explicit_id.as_deref() { extra.insert( "explicit_id".to_owned(), @@ -346,7 +352,7 @@ impl State<'_, '_> { parent.as_deref(), extra, ); - self.register_heading_target(title, explicit_id.as_deref(), &id); + self.register_heading_target(&anchor_slug, explicit_id.as_deref(), &id); self.collect_inline_descendants(node, &id); self.heading_stack.push(HeadingFrame { level, @@ -408,6 +414,24 @@ impl State<'_, '_> { } else if kind == "pipe_table_cell" { extra.insert("table_role".to_owned(), Value::String("cell".to_owned())); } + if let Some(section) = self.heading_stack.last() { + extra.insert( + "document_section".to_owned(), + Value::String(section.qualified_name.clone()), + ); + extra.insert( + "qualified_name".to_owned(), + Value::String(format!( + "{}::{kind}#{}", + section.qualified_name, self.next_block_index + )), + ); + } else { + extra.insert( + "qualified_name".to_owned(), + Value::String(format!("{}::{kind}#{}", self.stem, self.next_block_index)), + ); + } let id = self.add_block_node( id, &label, @@ -928,8 +952,26 @@ impl State<'_, '_> { owner } - fn register_heading_target(&mut self, title: &str, explicit_id: Option<&str>, id: &str) { - let mut keys = vec![slugify(title)]; + fn heading_anchor_slug(&mut self, title: &str, explicit_id: Option<&str>) -> String { + let base = slugify(explicit_id.unwrap_or(title)); + if explicit_id.is_some() || base.is_empty() { + return base; + } + let occurrence = self + .heading_slug_occurrences + .entry(base.clone()) + .or_default(); + let slug = if *occurrence == 0 { + base + } else { + format!("{base}-{occurrence}") + }; + *occurrence = occurrence.saturating_add(1); + slug + } + + fn register_heading_target(&mut self, anchor_slug: &str, explicit_id: Option<&str>, id: &str) { + let mut keys = vec![anchor_slug.to_ascii_lowercase()]; if let Some(explicit_id) = explicit_id { keys.push(explicit_id.to_ascii_lowercase()); keys.push(slugify(explicit_id)); @@ -1004,13 +1046,13 @@ impl State<'_, '_> { .split_once('?') .map_or(path_part, |(path, _)| path) .trim(); - let target_path = if path_part.is_empty() { + let root_relative = path_part.starts_with('/'); + let unresolved_path = if path_part.is_empty() { lexical_normalize(self.path) + } else if root_relative { + lexical_normalize(Path::new(path_part.trim_start_matches('/'))) } else { - let mut target = PathBuf::from(path_part); - if target.extension().is_none() { - target.set_extension("md"); - } + let target = PathBuf::from(path_part); if target.is_absolute() { target } else { @@ -1023,14 +1065,20 @@ impl State<'_, '_> { ) } }; + let extension_inferred = unresolved_path.extension().is_none(); + let mut target_path = unresolved_path.clone(); + if extension_inferred { + target_path.set_extension("md"); + } let same_file = lexical_normalize(self.path) == target_path; if same_file { if let Some(fragment) = fragment.filter(|fragment| !fragment.is_empty()) { - let key = fragment.to_ascii_lowercase(); + let fragment_key = decode_fragment(fragment); + let key = fragment_key.to_ascii_lowercase(); let candidates = self .heading_targets .get(&key) - .or_else(|| self.heading_targets.get(&slugify(fragment))); + .or_else(|| self.heading_targets.get(&slugify(&fragment_key))); match candidates { Some(candidates) if candidates.len() == 1 => { if let Some(target_id) = candidates.first() { @@ -1047,9 +1095,6 @@ impl State<'_, '_> { } continue; } - if is_documentable_source(&target_path) && !target_path.is_file() { - continue; - } if !is_supported_local_link(&target_path) { continue; } @@ -1059,7 +1104,16 @@ impl State<'_, '_> { } else { "references" }; - self.add_link_edge(&pending, target_id, Some((relation, fragment))); + self.add_link_edge_with_hint( + &pending, + target_id, + Some((relation, fragment)), + DocumentTargetHint { + path: unresolved_path.to_string_lossy().replace('\\', "/"), + extension_inferred, + root_relative, + }, + ); } } @@ -1080,6 +1134,28 @@ impl State<'_, '_> { ); } + fn add_link_edge_with_hint( + &mut self, + pending: &PendingLink, + target: String, + relation: Option<(&str, Option<&str>)>, + hint: DocumentTargetHint, + ) { + self.add_link_edge(pending, target, relation); + if let Some(edge) = self.extraction.edges.last_mut() { + edge.attributes + .insert("_document_target_path".to_owned(), Value::String(hint.path)); + edge.attributes.insert( + "_document_target_extension_inferred".to_owned(), + Value::Bool(hint.extension_inferred), + ); + edge.attributes.insert( + "_document_target_root_relative".to_owned(), + Value::Bool(hint.root_relative), + ); + } + } + fn add_unresolved(&mut self, pending: &PendingLink, reason: &str, target: &str) { if self.unresolved_links.len() >= MAX_DIAGNOSTICS { return; @@ -1648,6 +1724,34 @@ fn slugify(text: &str) -> String { slug } +fn decode_fragment(fragment: &str) -> String { + let bytes = fragment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len().min(512)); + let mut index = 0; + while index < bytes.len() && decoded.len() < 512 { + if bytes[index] == b'%' + && let (Some(high), Some(low)) = (bytes.get(index + 1), bytes.get(index + 2)) + && let (Some(high), Some(low)) = (hex_digit(*high), hex_digit(*low)) + { + decoded.push(high.saturating_mul(16).saturating_add(low)); + index = index.saturating_add(3); + continue; + } + decoded.push(bytes[index]); + index = index.saturating_add(1); + } + String::from_utf8_lossy(&decoded).into_owned() +} + +const fn hex_digit(value: u8) -> Option { + match value { + b'0'..=b'9' => Some(value - b'0'), + b'a'..=b'f' => Some(value - b'a' + 10), + b'A'..=b'F' => Some(value - b'A' + 10), + _ => None, + } +} + fn normalize_reference_label(label: &str) -> String { label .trim() diff --git a/crates/compass-languages/tests/markdown_coverage.rs b/crates/compass-languages/tests/markdown_coverage.rs index 7ab5918e..e2e0cede 100644 --- a/crates/compass-languages/tests/markdown_coverage.rs +++ b/crates/compass-languages/tests/markdown_coverage.rs @@ -231,6 +231,10 @@ let hidden = "[not a link](ignored.md)"; edge.attributes.get("fragment") == Some(&serde_json::json!("start")) && edge.target != root.id })); + assert!(extraction.nodes.iter().any(|node| { + node.attributes.get("document_kind") == Some(&serde_json::json!("paragraph")) + && node.string("qualified_name").contains("::paragraph#") + })); assert!(extraction.nodes.iter().all(|node| { node.attributes .get("start_byte") @@ -257,18 +261,47 @@ let hidden = "[not a link](ignored.md)"; } #[test] -fn markdown_duplicate_fragments_are_explicitly_unresolved() -> Result<(), Box> { +fn markdown_duplicate_heading_slugs_follow_source_order_and_explicit_ids_remain_ambiguous() +-> Result<(), Box> { let extraction = Engine::default().extract_source( std::path::Path::new("guide.md"), - b"# Same\n\n## Same\n\n## Other\n\n[jump](#same)\n", + b"# Same\n\n## Same\n\n## Other\n\n[first](#same) [second](#same-1)\n", )?; - assert!( - !extraction - .edges - .iter() - .any(|edge| edge.attributes.get("link_kind") == Some(&serde_json::json!("inline"))) - ); - let unresolved = extraction + let links = extraction + .edges + .iter() + .filter(|edge| edge.attributes.get("link_kind") == Some(&serde_json::json!("inline"))) + .collect::>(); + assert_eq!(links.len(), 2); + assert_ne!(links[0].target, links[1].target); + let slugs = extraction + .nodes + .iter() + .filter_map(|node| node.attributes.get("anchor_slug")) + .filter_map(serde_json::Value::as_str) + .collect::>(); + assert!(slugs.contains(&"same")); + assert!(slugs.contains(&"same-1")); + + let encoded = Engine::default().extract_source( + std::path::Path::new("encoded.md"), + b"# Encoded {#agent:rules}\n\n[jump](#agent%3Arules)\n", + )?; + let encoded_heading = encoded + .nodes + .iter() + .find(|node| node.string("explicit_id") == "agent:rules") + .ok_or("missing encoded-fragment heading")?; + assert!(encoded.edges.iter().any(|edge| { + edge.attributes.get("fragment") == Some(&serde_json::json!("agent%3Arules")) + && edge.target == encoded_heading.id + })); + + let explicit = Engine::default().extract_source( + std::path::Path::new("explicit.md"), + b"# One {#shared}\n\n# Two {#shared}\n\n[jump](#shared)\n", + )?; + let unresolved = explicit .extensions .get("markdown_unresolved_links") .and_then(serde_json::Value::as_array) diff --git a/crates/compass-resolve/src/lib.rs b/crates/compass-resolve/src/lib.rs index c3ba6b4b..d62de5a3 100644 --- a/crates/compass-resolve/src/lib.rs +++ b/crates/compass-resolve/src/lib.rs @@ -1162,6 +1162,8 @@ fn finish_resolution( &mut language_facts.calls, ); profile_internal("resolver collision disambiguation", &mut profile_started); + resolve_document_link_targets(&mut merged, &canonical_root); + profile_internal("resolver document links", &mut profile_started); if has_javascript { resolve_javascript_workspace_symbols(&mut merged); } @@ -4193,6 +4195,9 @@ fn canonicalize_file_targets(extraction: &mut Extraction, root: &Path) { }) .collect::>(); for edge in &mut extraction.edges { + if edge.attributes.contains_key("_document_target_path") { + continue; + } if node_ids.contains(edge.target.as_str()) { continue; } @@ -4211,6 +4216,276 @@ fn canonicalize_file_targets(extraction: &mut Extraction, root: &Path) { } } +const DOCUMENT_TARGET_EXTENSIONS: [&str; 5] = ["md", "markdown", "mdx", "qmd", "skill"]; + +/// Resolve Markdown links only after the complete project inventory is known. +/// +/// The per-file extractor preserves the source spelling and an exact wiring +/// site. This stage can therefore resolve cross-file fragments, extensionless +/// links, repository-root links, directory index documents, and unique wiki +/// stems without filesystem-order guesses or cross-language policy. +fn resolve_document_link_targets(extraction: &mut Extraction, root: &Path) { + let profile = std::env::var_os("COMPASS_PROFILE_INTERNAL").is_some(); + let mut considered = 0_usize; + let mut rewritten = 0_usize; + let mut roots_by_source = BTreeMap::>::new(); + let mut wiki_stems = BTreeMap::>::new(); + let mut headings = BTreeMap::<(String, String), Vec>::new(); + + for node in &extraction.nodes { + let source = string_attribute(node, "source_file"); + if source.is_empty() { + continue; + } + let source = source_key(&source, root); + let document_root = + node.string("file_type") == "document" && node.string("document_kind") == "document"; + if document_root || is_file_node(node, &source) { + roots_by_source + .entry(source.clone()) + .or_default() + .push(node.id.clone()); + if document_root + && let Some(stem) = Path::new(&source) + .file_stem() + .and_then(|value| value.to_str()) + { + wiki_stems + .entry(stem.to_ascii_lowercase()) + .or_default() + .push((source.clone(), node.id.clone())); + } + } + if node.string("file_type") == "document" && node.string("document_kind") == "heading" { + let mut aliases = BTreeSet::new(); + for attribute in ["anchor_slug", "explicit_id"] { + let value = node.string(attribute); + if !value.is_empty() { + aliases.insert(value.to_ascii_lowercase()); + } + } + for alias in aliases { + headings + .entry((source.clone(), alias)) + .or_default() + .push(node.id.clone()); + } + } + } + for candidates in roots_by_source.values_mut() { + candidates.sort(); + candidates.dedup(); + } + for candidates in wiki_stems.values_mut() { + candidates.sort(); + candidates.dedup(); + } + for candidates in headings.values_mut() { + candidates.sort(); + candidates.dedup(); + } + + for edge in &mut extraction.edges { + let Some(target_path) = edge + .attributes + .get("_document_target_path") + .and_then(Value::as_str) + .filter(|path| !path.is_empty()) + .map(str::to_owned) + else { + continue; + }; + considered = considered.saturating_add(1); + let extension_inferred = edge + .attributes + .get("_document_target_extension_inferred") + .and_then(Value::as_bool) + .unwrap_or(false); + let wiki_link = + edge.attributes.get("link_kind").and_then(Value::as_str) == Some("wikilink"); + + let mut source_candidates = BTreeSet::new(); + let normalized = document_target_key(&target_path, root); + if !normalized.is_empty() { + source_candidates.insert(normalized.clone()); + } + if extension_inferred { + let path = Path::new(&normalized); + for extension in DOCUMENT_TARGET_EXTENSIONS { + source_candidates.insert( + path.with_extension(extension) + .to_string_lossy() + .replace('\\', "/"), + ); + source_candidates.insert( + path.join("README") + .with_extension(extension) + .to_string_lossy() + .replace('\\', "/"), + ); + source_candidates.insert( + path.join("index") + .with_extension(extension) + .to_string_lossy() + .replace('\\', "/"), + ); + } + } + + let mut candidates = source_candidates + .iter() + .filter_map(|source| { + roots_by_source + .get(source) + .filter(|targets| targets.len() == 1) + .and_then(|targets| targets.first()) + .map(|target| (source.clone(), target.clone())) + }) + .collect::>(); + if candidates.is_empty() && wiki_link { + let stem = Path::new(&normalized) + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + if let Some(targets) = wiki_stems.get(&stem).filter(|targets| targets.len() == 1) { + candidates.extend(targets.iter().cloned()); + } + } + candidates.sort(); + candidates.dedup(); + + let Some((target_source, document_target)) = + (candidates.len() == 1).then(|| candidates.pop()).flatten() + else { + let status = if candidates.is_empty() { + "missing_target" + } else { + "ambiguous_target" + }; + mark_unresolved_document_link(edge, &target_path, status); + continue; + }; + let fragment = edge + .attributes + .get("fragment") + .and_then(Value::as_str) + .filter(|fragment| !fragment.is_empty()) + .map(str::to_owned); + let target = if let Some(fragment) = fragment.as_deref() { + let key = decode_markdown_fragment(fragment).to_ascii_lowercase(); + let Some(targets) = headings.get(&(target_source, key)) else { + mark_unresolved_document_link(edge, &target_path, "missing_fragment"); + continue; + }; + if targets.len() != 1 { + mark_unresolved_document_link(edge, &target_path, "ambiguous_fragment"); + continue; + } + targets[0].clone() + } else { + document_target + }; + if target != edge.target { + edge.attributes.insert( + "_document_original_target".to_owned(), + Value::String(edge.target.clone()), + ); + edge.target = target; + rewritten = rewritten.saturating_add(1); + } + edge.attributes.insert( + "rule".to_owned(), + Value::String("document-link-exact-target".to_owned()), + ); + edge.attributes.insert( + "resolution_rule".to_owned(), + Value::String("document-link-target-resolution".to_owned()), + ); + } + if profile { + eprintln!( + "[compass internal] document links considered={considered} rewritten={rewritten}" + ); + } +} + +fn document_target_key(source: &str, root: &Path) -> String { + let key = source_key(source, root); + if !Path::new(&key).is_absolute() { + return key; + } + let path = Path::new(source); + let Some((parent, file_name)) = path.parent().zip(path.file_name()) else { + return key; + }; + let Ok(parent) = std::fs::canonicalize(parent) else { + return key; + }; + let Ok(parent) = parent.strip_prefix(root) else { + return key; + }; + parent.join(file_name).to_string_lossy().replace('\\', "/") +} + +fn mark_unresolved_document_link(edge: &mut EdgeRecord, target_path: &str, status: &str) { + let fragment = edge + .attributes + .get("fragment") + .and_then(Value::as_str) + .unwrap_or_default(); + let source_file = edge + .attributes + .get("source_file") + .and_then(Value::as_str) + .unwrap_or_default(); + let start_byte = edge + .attributes + .get("start_byte") + .and_then(Value::as_u64) + .unwrap_or_default() + .to_string(); + edge.target = make_id(&[ + "unresolved_document_link", + source_file, + &start_byte, + target_path, + fragment, + ]); + edge.attributes.insert( + "_document_target_resolution".to_owned(), + Value::String(status.to_owned()), + ); +} + +fn decode_markdown_fragment(fragment: &str) -> String { + let bytes = fragment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len().min(512)); + let mut index = 0; + while index < bytes.len() && decoded.len() < 512 { + if bytes[index] == b'%' + && let (Some(high), Some(low)) = (bytes.get(index + 1), bytes.get(index + 2)) + && let (Some(high), Some(low)) = (hex_digit(*high), hex_digit(*low)) + { + decoded.push(high.saturating_mul(16).saturating_add(low)); + index = index.saturating_add(3); + continue; + } + decoded.push(bytes[index]); + index = index.saturating_add(1); + } + String::from_utf8_lossy(&decoded).into_owned() +} + +const fn hex_digit(value: u8) -> Option { + match value { + b'0'..=b'9' => Some(value - b'0'), + b'a'..=b'f' => Some(value - b'a' + 10), + b'A'..=b'F' => Some(value - b'A' + 10), + _ => None, + } +} + fn rewire_unique_stub_nodes(extraction: &mut Extraction) { let normalized_label = |node: &NodeRecord| { node.label() diff --git a/crates/compass-resolve/tests/universal_evidence.rs b/crates/compass-resolve/tests/universal_evidence.rs index 37976c4a..ba315dee 100644 --- a/crates/compass-resolve/tests/universal_evidence.rs +++ b/crates/compass-resolve/tests/universal_evidence.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::error::Error; use std::path::Path; -use compass_graph::{BuildEvidence, normalize_v1}; +use compass_graph::{BuildEvidence, InferenceLevel, apply_inference_level, normalize_v1}; use compass_languages::{CandidateRelation, Engine, EvidenceLimits, validate_evidence}; use compass_model::code_graph::NodeKind; use compass_resolve::evidence::{ @@ -237,6 +237,265 @@ fn markdown_documents_edges_resolve_to_universal_file_inventory() -> Result<(), Ok(()) } +#[test] +fn markdown_project_links_resolve_exact_files_fragments_indexes_and_unique_wiki_stems() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let docs = directory.path().join("docs"); + let topic = docs.join("topic"); + std::fs::create_dir_all(&topic)?; + let guide_path = docs.join("guide.md"); + let reference_path = docs.join("reference.md"); + let index_path = topic.join("README.md"); + let ambiguous_markdown_path = docs.join("dual.md"); + let ambiguous_mdx_path = docs.join("dual.mdx"); + let skill_path = docs.join("agent.skill"); + let code_paths = [ + (directory.path().join("src/module.py"), "python"), + (directory.path().join("src/lib.rs"), "rust"), + (directory.path().join("src/main.go"), "go"), + (directory.path().join("src/Main.java"), "java"), + (directory.path().join("src/index.ts"), "typescript"), + ]; + std::fs::create_dir_all(code_paths[0].0.parent().ok_or("missing code parent")?)?; + std::fs::write( + &guide_path, + concat!( + "# Guide\n", + "[exact](reference.md#target-section)\n", + "[extensionless](reference#target-section)\n", + "[root](/docs/reference.md#target-section)\n", + "[encoded](reference.md#target%2Dsection)\n", + "[[Reference#target-section]]\n", + "[directory](topic/#overview)\n", + "[skill](agent#rules)\n", + "[python](../src/module.py) [rust](../src/lib.rs) [go](../src/main.go)\n", + "[java](../src/Main.java) [typescript](../src/index.ts)\n", + "[missing fragment](reference.md#absent)\n", + "[ambiguous extension](dual#section)\n", + ), + )?; + std::fs::write( + &reference_path, + "# Reference\n\n## Target section\n\nImportant behavior.\n", + )?; + std::fs::write(&index_path, "# Overview\n\nDirectory documentation.\n")?; + std::fs::write(&ambiguous_markdown_path, "# Section\n")?; + std::fs::write(&ambiguous_mdx_path, "# Section\n")?; + std::fs::write(&skill_path, "# Agent\n\n## Rules\n\nUse exact evidence.\n")?; + for (path, source) in code_paths.iter().zip([ + "def documented():\n pass\n", + "pub fn documented() {}\n", + "package main\nfunc documented() {}\n", + "class Main { void documented() {} }\n", + "export function documented() {}\n", + ]) { + std::fs::write(&path.0, source)?; + } + + let mut engine = Engine::default(); + let mut extractions = vec![ + engine.extract(&guide_path)?, + engine.extract(&reference_path)?, + engine.extract(&index_path)?, + engine.extract(&ambiguous_markdown_path)?, + engine.extract(&ambiguous_mdx_path)?, + engine.extract(&skill_path)?, + ]; + for (path, _) in &code_paths { + extractions.push(engine.extract(path)?); + } + let merged = resolve_with_root(&extractions, &HashMap::new(), directory.path()); + let target_heading = merged + .nodes + .iter() + .find(|node| { + node.string("source_file").ends_with("docs/reference.md") + && node.string("anchor_slug") == "target-section" + }) + .ok_or("missing cross-document target heading")?; + let overview_heading = merged + .nodes + .iter() + .find(|node| { + node.string("source_file").ends_with("docs/topic/README.md") + && node.string("anchor_slug") == "overview" + }) + .ok_or("missing directory index heading")?; + let skill_heading = merged + .nodes + .iter() + .find(|node| { + node.string("source_file").ends_with("docs/agent.skill") + && node.string("anchor_slug") == "rules" + }) + .ok_or("missing skill heading")?; + let resolved = merged + .edges + .iter() + .filter(|edge| { + edge.string("source_file").ends_with("docs/guide.md") + && edge.string("relation") == "references" + && edge.string("_document_target_resolution").is_empty() + }) + .collect::>(); + assert_eq!( + resolved.len(), + 7, + "resolved={resolved:#?}; guide_edges={:#?}", + merged + .edges + .iter() + .filter(|edge| edge.string("source_file").ends_with("docs/guide.md")) + .collect::>() + ); + assert_eq!( + resolved + .iter() + .filter(|edge| edge.target == target_heading.id) + .count(), + 5 + ); + assert_eq!( + resolved + .iter() + .filter(|edge| edge.target == overview_heading.id) + .count(), + 1 + ); + assert_eq!( + resolved + .iter() + .filter(|edge| edge.target == skill_heading.id) + .count(), + 1 + ); + assert!(resolved.iter().all(|edge| { + edge.string("rule") == "document-link-exact-target" + && edge.string("resolution_rule") == "document-link-target-resolution" + && !edge + .attributes + .contains_key(compass_model::provenance::ENDPOINT_REWRITE_RULES_ATTRIBUTE) + })); + let unresolved = merged + .edges + .iter() + .filter(|edge| { + edge.string("source_file").ends_with("docs/guide.md") + && !edge.string("_document_target_resolution").is_empty() + }) + .map(|edge| edge.string("_document_target_resolution")) + .collect::>(); + assert_eq!(unresolved, ["missing_fragment", "ambiguous_target"]); + let code_file_ids = merged + .nodes + .iter() + .filter(|node| { + node.string("source_file").contains("src/") && node.string("symbol_kind") == "file" + }) + .map(|node| node.id.as_str()) + .collect::>(); + assert_eq!( + code_file_ids.len(), + 5, + "missing language file inventory node" + ); + assert_eq!( + merged + .edges + .iter() + .filter(|edge| { + edge.string("source_file").ends_with("docs/guide.md") + && edge.string("relation") == "documents" + && code_file_ids.contains(edge.target.as_str()) + }) + .count(), + 5 + ); + let graph = compass_graph::build(&[merged], true, true, Some(directory.path()))?; + let mut published = compass_graph::normalize_document_v1_with_inventory_best_effort( + &graph, + directory.path(), + "test", + None, + { + let mut inventory = [ + (&guide_path, "markdown"), + (&reference_path, "markdown"), + (&index_path, "markdown"), + (&ambiguous_markdown_path, "markdown"), + (&ambiguous_mdx_path, "markdown"), + (&skill_path, "markdown"), + ] + .into_iter() + .map(|(path, language)| compass_graph::InventoryEvidence { + path: path.clone(), + language: Some(language.to_owned()), + producer: format!("compass.languages.{language}"), + status: compass_model::code_graph::ExtractionStatus::Extracted, + reason: None, + }) + .collect::>(); + inventory.extend(code_paths.iter().map(|(path, language)| { + compass_graph::InventoryEvidence { + path: path.clone(), + language: Some((*language).to_owned()), + producer: format!("compass.languages.{language}"), + status: compass_model::code_graph::ExtractionStatus::Extracted, + reason: None, + } + })); + inventory + }, + )?; + let published_links = published + .document + .links + .iter() + .filter(|edge| { + edge.kind.as_str() == "references" + && edge + .relationship_site + .as_ref() + .is_some_and(|site| site.file == "docs/guide.md") + }) + .count(); + assert_eq!(published_links, 7, "published={:#?}", published.document); + assert_eq!( + published + .document + .links + .iter() + .filter(|edge| { + edge.kind.as_str() == "documents" + && edge + .relationship_site + .as_ref() + .is_some_and(|site| site.file == "docs/guide.md") + }) + .count(), + 5 + ); + apply_inference_level(&mut published.document, InferenceLevel::Low); + assert_eq!( + published + .document + .links + .iter() + .filter(|edge| { + matches!(edge.kind.as_str(), "references" | "documents") + && edge + .relationship_site + .as_ref() + .is_some_and(|site| site.file == "docs/guide.md") + }) + .count(), + 12, + "exact document relationships must survive the default low inference level" + ); + Ok(()) +} + #[test] fn collection_resolution_consumes_each_rust_evidence_batch_once() -> Result<(), Box> { let mut engine = Engine::default(); diff --git a/docs/design/document-processing.md b/docs/design/document-processing.md index e224e8dc..80317428 100644 --- a/docs/design/document-processing.md +++ b/docs/design/document-processing.md @@ -40,6 +40,9 @@ compass-languages::Engine document root + structural blocks + link evidence | v +compass-resolve project inventory + heading targets + | + v compass-graph / compass-core publication ``` @@ -68,9 +71,12 @@ The structural projection emits ordered nodes for headings, paragraphs, lists and list items, block quotes, thematic breaks, fenced and indented code, pipe-table containers/headers/rows/cells, HTML blocks, and reference definitions. Each node carries `document_kind`, `block_index`, source identity, -and an exact byte range. Heading nodes additionally carry `heading_level`, -`heading_style`, `qualified_name`, and a deterministic `anchor_slug`; headings -with `{#explicit-id}` retain `explicit_id`. +a section-qualified `qualified_name`, and an exact byte range. Blocks beneath a +heading also carry `document_section`. Heading nodes additionally carry +`heading_level`, `heading_style`, and a deterministic `anchor_slug`; repeated +automatic slugs receive source-order `-1`, `-2`, … suffixes, while headings +with `{#explicit-id}` retain `explicit_id` and duplicate explicit IDs remain +ambiguous. Nested blocks are represented by `contains` edges. Inline links are owned by the smallest containing structural block, so a link in a list item or table @@ -95,11 +101,21 @@ lossy display representation while offsets remain byte offsets). ## Link resolution Compass records external links as bounded `markdown_external_links` evidence -without fetching them. Supported local links produce `references` edges, while -links to an existing source document may use the `documents` relation. A -fragment-only link resolves to a heading only when its slug or explicit ID is -unique. Duplicate or missing fragments remain in `markdown_unresolved_links` -with an explicit reason; extraction never selects the first same-named heading. +without fetching them. The per-file extractor preserves local link spelling +and its exact source site; the project resolver selects a target only against +the complete extracted inventory. Supported local links produce `references` +edges, while links to source code use `documents` and target that language's +file-inventory node. + +Exact and repository-root paths, `.md`/`.markdown`/`.mdx`/`.qmd`/`.skill` extension +inference, directory `README`/`index` documents, and unique wikilink stems are +bounded resolution rules. A same-file or cross-file fragment is percent-decoded +within a fixed bound and resolves only when its slug or explicit ID is unique. +Duplicate or missing fragments and +ambiguous extension or stem candidates remain unresolved; resolution never +selects the first candidate or substitutes a document root. Exact resolved +relationships retain source-backed confidence and are therefore available at +the default low inference level. Reference definitions and usages retain separate source sites. Wikilinks, autolinks, email links, inline links, and reference links carry a `link_kind`, diff --git a/docs/reference/document-formats.md b/docs/reference/document-formats.md index e0d27dcc..09aa731b 100644 --- a/docs/reference/document-formats.md +++ b/docs/reference/document-formats.md @@ -9,8 +9,8 @@ alone. | Construct | Current behavior | Provenance | | --- | --- | --- | -| ATX / Setext headings | Heading node, hierarchy, slug, optional explicit ID | exact node range | -| Paragraphs and inline text | Paragraph block | exact node range | +| ATX / Setext headings | Heading node, hierarchy, source-order duplicate slug, optional explicit ID | exact node range | +| Paragraphs and inline text | Section-qualified paragraph block | exact node range | | Lists and task items | List/list-item blocks; `task_checked` when present | `contains` + exact ranges | | Block quotes and thematic breaks | Structural block | exact node range | | Fenced / indented code | Code block; fenced info string becomes `language` | exact node range | @@ -33,7 +33,9 @@ Tree-sitter block and inline grammars. Supported source extensions are Document and block nodes retain the common graph fields `id`, `label`, `file_type`, `document_kind`, `source_file`, `_origin`, `start_byte`, `end_byte`, `start_line`, `end_line`, `column_start`, and `column_end` where -applicable. Markdown-specific root extensions include: +applicable. Structural blocks also carry a deterministic `qualified_name` and, +when nested under a heading, `document_section`. Markdown-specific root +extensions include: - `markdown_block_count`; - `markdown_link_count`; @@ -55,10 +57,15 @@ attributes and must not parse stable IDs as path components. ### Link boundary -External links are recorded as evidence but never fetched. Same-file fragments -resolve only to a unique heading slug or explicit ID. Ambiguous and missing -fragments are explicit unresolved evidence. Unsupported local suffixes and -missing document targets do not create invented nodes. +External links are recorded as evidence but never fetched. Same-file and +cross-file fragments are percent-decoded within a fixed bound and resolve only +to a unique heading slug or explicit ID. +Project resolution supports exact paths, `.md`/`.markdown`/`.mdx`/`.qmd`/`.skill` +extension inference, repository-root paths, directory `README`/`index` +documents, and unique wikilink stems. It can also connect documentation to the +file-inventory node for any extracted source language. Ambiguous or missing +targets and fragments remain unresolved; Compass does not select the first +candidate, fall back to a document root, or invent a node. ## HTML