From 61d7b0dd55313076a081bad823eb69796d4b8426 Mon Sep 17 00:00:00 2001 From: phernandez Date: Fri, 31 Jul 2026 22:50:43 -0500 Subject: [PATCH] Add portable document ingestion contracts Signed-off-by: phernandez --- src/basic_memory/markdown/entity_parser.py | 13 +- .../runtime/note_object_metadata.py | 4 +- src/basic_memory/schemas/__init__.py | 62 ++ src/basic_memory/schemas/document.py | 826 +++++++++++++++ tests/markdown/test_entity_parser.py | 21 + tests/schemas/test_document.py | 961 ++++++++++++++++++ tests/test_runtime.py | 4 + 7 files changed, 1888 insertions(+), 3 deletions(-) create mode 100644 src/basic_memory/schemas/document.py create mode 100644 tests/schemas/test_document.py diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 72a3e2365..d08b3a0e8 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -322,9 +322,18 @@ async def parse_markdown_content( if tags: metadata["tags"] = tags - # Parse content for observations and relations + # Raw extraction text is useful for retrieval but is untrusted semantic input. + # The ingestion contract sets this explicit opt-out so arbitrary PDF syntax cannot + # mint observations or relations before the bounded enrichment pass. entity_frontmatter = EntityFrontmatter(metadata=metadata) - entity_content = parse(post.content) + semantic_setting = metadata.get("bm_parse_semantics") + parse_semantics = not ( + semantic_setting is False + or (isinstance(semantic_setting, str) and semantic_setting.lower() == "false") + ) + entity_content = ( + parse(post.content) if parse_semantics else EntityContent(content=post.content) + ) # Canonical frontmatter timestamps describe note semantics. File times are # only compatibility fallbacks for notes that do not declare them. diff --git a/src/basic_memory/runtime/note_object_metadata.py b/src/basic_memory/runtime/note_object_metadata.py index 7ee8f2713..350cd7947 100644 --- a/src/basic_memory/runtime/note_object_metadata.py +++ b/src/basic_memory/runtime/note_object_metadata.py @@ -40,8 +40,10 @@ # collaboration_relay = a relay service persisting a live collaboration # document with its service credential (issue #1445); the webhook-canonical # note.updated event echoes this source as the write's actor origin. +# document_ingestion = a hosted worker accepting a deterministic document +# extraction or ingestion-run note through the canonical note mutation path. VALID_NOTE_OBJECT_SOURCES: frozenset[RuntimeNoteChangeSource] = frozenset( - {"api", "collaboration_relay", "mcp", "s3_webhook", "web_v2"} + {"api", "collaboration_relay", "document_ingestion", "mcp", "s3_webhook", "web_v2"} ) # Named because the accepted-note write path special-cases relay writes: the # relay superseding its own prior write is never a real conflict (#1589). diff --git a/src/basic_memory/schemas/__init__.py b/src/basic_memory/schemas/__init__.py index 8556c2bd4..b4f734dd3 100644 --- a/src/basic_memory/schemas/__init__.py +++ b/src/basic_memory/schemas/__init__.py @@ -55,6 +55,38 @@ DirectoryNode, ) +from basic_memory.schemas.document import ( + DocumentAgentObservationV1, + DocumentAgentOutputV1, + DocumentAgentRelationV1, + DocumentExtractionStatus, + DocumentExtractionV1, + DocumentIngestionFailureV1, + DocumentIngestionRunFrontmatterV1, + DocumentIngestionRunMarkdownV1, + DocumentIngestionRunOutputV1, + DocumentIngestionRunStateV1, + DocumentIngestionStage, + DocumentIngestionV1, + DocumentMarkdownV1, + DocumentMetadataV1, + DocumentNoteFrontmatterV1, + DocumentRevisionKind, + DocumentRevisionReferenceV1, + DocumentSourceV1, + assemble_document_ingestion_run_markdown, + assemble_document_markdown, + derive_document_ingestion_run_id, + derive_document_ingestion_run_path, + derive_document_note_external_id, + derive_document_note_path, + document_markdown_checksum, + enrich_document_markdown, + parse_document_ingestion_run_markdown, + parse_document_markdown, + require_document_ingestion_transition, +) + # For convenient imports, export all models __all__ = [ # Base @@ -88,4 +120,34 @@ "ProjectIndexStatusResponse", # Directory "DirectoryNode", + # Document ingestion + "DocumentAgentObservationV1", + "DocumentAgentOutputV1", + "DocumentAgentRelationV1", + "DocumentExtractionStatus", + "DocumentExtractionV1", + "DocumentIngestionFailureV1", + "DocumentIngestionRunFrontmatterV1", + "DocumentIngestionRunMarkdownV1", + "DocumentIngestionRunOutputV1", + "DocumentIngestionRunStateV1", + "DocumentIngestionStage", + "DocumentIngestionV1", + "DocumentMarkdownV1", + "DocumentMetadataV1", + "DocumentNoteFrontmatterV1", + "DocumentRevisionKind", + "DocumentRevisionReferenceV1", + "DocumentSourceV1", + "assemble_document_ingestion_run_markdown", + "assemble_document_markdown", + "derive_document_ingestion_run_id", + "derive_document_ingestion_run_path", + "derive_document_note_external_id", + "derive_document_note_path", + "document_markdown_checksum", + "enrich_document_markdown", + "parse_document_ingestion_run_markdown", + "parse_document_markdown", + "require_document_ingestion_transition", ] diff --git a/src/basic_memory/schemas/document.py b/src/basic_memory/schemas/document.py new file mode 100644 index 000000000..038a7eda1 --- /dev/null +++ b/src/basic_memory/schemas/document.py @@ -0,0 +1,826 @@ +"""Portable contracts for parser-neutral document ingestion. + +The original binary remains a file entity. These schemas describe the trusted Markdown +projection created from that source and the optional ingestion-run note that records its +lifecycle. Parser execution, storage versions, queues, and agent providers belong to the +runtime that composes these contracts. +""" + +import hashlib +import json +import re +from datetime import date, datetime +from enum import StrEnum +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Annotated, Literal +from uuid import NAMESPACE_URL, UUID, uuid5 + +from frontmatter import Post +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictInt, + StrictStr, + StringConstraints, + field_validator, + model_validator, +) + +from basic_memory.file_utils import dump_frontmatter, parse_frontmatter, remove_frontmatter + +if TYPE_CHECKING: # pragma: no cover - static import only + from basic_memory.markdown.entity_parser import EntityContent + + +type NonEmptyText = Annotated[ + StrictStr, + StringConstraints(strip_whitespace=True, min_length=1), +] +type Sha256Checksum = Annotated[ + StrictStr, + StringConstraints(pattern=r"^sha256:[0-9a-f]{64}$"), +] +type AgentTag = Annotated[ + StrictStr, + StringConstraints(pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]*$"), +] + +_DOCUMENT_INGESTION_NAMESPACE = uuid5( + NAMESPACE_URL, + "https://basicmemory.com/schemas/document-ingestion/v1", +) +_SHA256_CHECKSUM_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") + + +class _DocumentContractModel(BaseModel): + """Fail-fast base for trusted ingestion boundaries.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class DocumentExtractionStatus(StrEnum): + """Outcome reported by a parser-neutral extraction provider.""" + + complete = "complete" + partial = "partial" + needs_ocr = "needs_ocr" + failed = "failed" + + +class DocumentIngestionStage(StrEnum): + """Lifecycle stage of the stable derived document note.""" + + raw = "raw" + ready = "ready" + needs_review = "needs_review" + failed = "failed" + + +class DocumentRevisionKind(StrEnum): + """Meaning of a version referenced by an ingestion-run note.""" + + raw_extraction = "raw_extraction" + agent_enriched = "agent_enriched" + human_edit = "human_edit" + + +class DocumentSourceV1(_DocumentContractModel): + """Stable provenance for the original binary file entity.""" + + kind: Literal["file"] = "file" + media_type: Annotated[ + StrictStr, + StringConstraints(pattern=r"^[a-z0-9][a-z0-9!#$&^_.+\-]*/[a-z0-9][a-z0-9!#$&^_.+\-]*$"), + ] + entity_external_id: UUID + file_path: NonEmptyText + checksum: Sha256Checksum + size_bytes: int = Field(ge=0, strict=True) + storage_version_id: NonEmptyText | None = None + storage_etag: NonEmptyText | None = None + + @field_validator("file_path") + @classmethod + def validate_file_path(cls, value: str) -> str: + return _validate_project_relative_path(value) + + +class DocumentExtractionV1(_DocumentContractModel): + """Parser-neutral extraction diagnostics stored on the document note.""" + + engine: NonEmptyText + engine_version: NonEmptyText + profile: NonEmptyText + options_hash: Sha256Checksum + classification: NonEmptyText | None = None + status: DocumentExtractionStatus + extracted_at: datetime + duration_ms: int | None = Field(default=None, ge=0, strict=True) + page_count: int = Field(ge=0, strict=True) + extracted_page_count: int = Field(ge=0, strict=True) + requires_ocr: StrictBool + ocr_page_count: int = Field(ge=0, strict=True) + pages_needing_ocr: tuple[StrictInt, ...] = () + confidence: float | None = Field(default=None, ge=0.0, le=1.0, strict=True) + has_encoding_issues: StrictBool = False + has_tables: StrictBool = False + has_columns: StrictBool = False + + @field_validator("extracted_at") + @classmethod + def require_aware_extracted_at(cls, value: datetime) -> datetime: + return _require_aware_datetime(value, field_name="extracted_at") + + @model_validator(mode="after") + def validate_page_diagnostics(self) -> "DocumentExtractionV1": + if self.extracted_page_count > self.page_count: + raise ValueError("extracted_page_count cannot exceed page_count") + if self.ocr_page_count != len(self.pages_needing_ocr): + raise ValueError("ocr_page_count must equal the number of pages_needing_ocr") + if self.pages_needing_ocr != tuple(sorted(set(self.pages_needing_ocr))): + raise ValueError("pages_needing_ocr must be unique and sorted") + if any(page < 1 or page > self.page_count for page in self.pages_needing_ocr): + raise ValueError("pages_needing_ocr must contain valid one-based page numbers") + if self.requires_ocr != bool(self.pages_needing_ocr): + raise ValueError("requires_ocr must match whether pages_needing_ocr is non-empty") + if self.status is DocumentExtractionStatus.needs_ocr and not self.requires_ocr: + raise ValueError("needs_ocr extraction status requires at least one OCR page") + if self.status is DocumentExtractionStatus.complete: + if self.requires_ocr or self.extracted_page_count != self.page_count: + raise ValueError("complete extraction must cover every page without OCR gaps") + return self + + +class DocumentIngestionV1(_DocumentContractModel): + """Trusted processing state for one version of the derived document note.""" + + stage: DocumentIngestionStage + pipeline_version: NonEmptyText + prompt_version: NonEmptyText | None = None + run_id: UUID + input_checksum: Sha256Checksum + base_checksum: Sha256Checksum | None = None + + @model_validator(mode="after") + def validate_base_checksum(self) -> "DocumentIngestionV1": + if self.stage is DocumentIngestionStage.raw and self.base_checksum is not None: + raise ValueError("raw ingestion stage cannot declare a base_checksum") + if self.stage is not DocumentIngestionStage.raw and self.base_checksum is None: + raise ValueError("non-raw ingestion stages require the raw base_checksum") + return self + + +class DocumentMetadataV1(_DocumentContractModel): + """Descriptive document fields that an enrichment pass may supply.""" + + kind: NonEmptyText | None = None + language: NonEmptyText | None = None + authors: tuple[NonEmptyText, ...] = () + published_at: date | None = None + + @field_validator("authors") + @classmethod + def require_unique_authors(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(value) != len(set(value)): + raise ValueError("authors must be unique") + return value + + +class DocumentNoteFrontmatterV1(_DocumentContractModel): + """Authoritative nested frontmatter for a ``type: document`` note.""" + + schema_version: Literal["1"] = "1" + title: NonEmptyText + type: Literal["document"] = "document" + schema_ref: Literal["schema/document-extraction"] = Field( + default="schema/document-extraction", + validation_alias=AliasChoices("schema", "schema_ref"), + serialization_alias="schema", + ) + tags: tuple[NonEmptyText, ...] = ("document",) + permalink: NonEmptyText | None = None + created: datetime | None = None + modified: datetime | None = None + source: DocumentSourceV1 + extraction: DocumentExtractionV1 + ingestion: DocumentIngestionV1 + document: DocumentMetadataV1 = Field(default_factory=DocumentMetadataV1) + bm_parse_semantics: StrictBool + + @field_validator("tags") + @classmethod + def require_unique_tags(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(value) != len(set(value)): + raise ValueError("tags must be unique") + return value + + @field_validator("created", "modified") + @classmethod + def require_aware_canonical_timestamps(cls, value: datetime | None) -> datetime | None: + if value is None: + return None + return _require_aware_datetime(value, field_name="canonical note timestamp") + + @model_validator(mode="after") + def validate_trusted_envelope(self) -> "DocumentNoteFrontmatterV1": + if self.source.checksum != self.ingestion.input_checksum: + raise ValueError("ingestion input_checksum must match the source checksum") + _require_deterministic_run_id( + source=self.source, + extraction=self.extraction, + run_id=self.ingestion.run_id, + pipeline_version=self.ingestion.pipeline_version, + prompt_version=self.ingestion.prompt_version, + ) + + parses_semantics = self.ingestion.stage in { + DocumentIngestionStage.ready, + DocumentIngestionStage.needs_review, + } + if self.bm_parse_semantics != parses_semantics: + raise ValueError( + "bm_parse_semantics must be false for raw/failed notes and true after enrichment" + ) + return self + + +class DocumentMarkdownV1(_DocumentContractModel): + """Validated document frontmatter paired with its Markdown body.""" + + frontmatter: DocumentNoteFrontmatterV1 + body: StrictStr + + +class DocumentRevisionReferenceV1(_DocumentContractModel): + """One exact Markdown revision referenced by an ingestion-run note.""" + + kind: DocumentRevisionKind + checksum: Sha256Checksum + storage_version_id: NonEmptyText | None = None + created_at: datetime + + @field_validator("created_at") + @classmethod + def require_aware_created_at(cls, value: datetime) -> datetime: + return _require_aware_datetime(value, field_name="created_at") + + +class DocumentIngestionFailureV1(_DocumentContractModel): + """Structured terminal failure details for an ingestion run.""" + + code: NonEmptyText + message: NonEmptyText + retryable: StrictBool + + +class DocumentIngestionRunStateV1(_DocumentContractModel): + """Lifecycle metadata for a queryable ingestion-run history note.""" + + run_id: UUID + stage: DocumentIngestionStage + pipeline_version: NonEmptyText + prompt_version: NonEmptyText | None = None + input_checksum: Sha256Checksum + started_at: datetime + completed_at: datetime | None = None + + @field_validator("started_at", "completed_at") + @classmethod + def require_aware_timestamps(cls, value: datetime | None) -> datetime | None: + if value is None: + return None + return _require_aware_datetime(value, field_name="ingestion run timestamp") + + @model_validator(mode="after") + def validate_completion(self) -> "DocumentIngestionRunStateV1": + if self.stage is DocumentIngestionStage.raw and self.completed_at is not None: + raise ValueError("raw ingestion run cannot be completed") + if self.stage is not DocumentIngestionStage.raw and self.completed_at is None: + raise ValueError("terminal ingestion run requires completed_at") + if self.completed_at is not None and self.completed_at < self.started_at: + raise ValueError("completed_at cannot precede started_at") + return self + + +class DocumentIngestionRunOutputV1(_DocumentContractModel): + """Stable document identity and exact revisions produced by an ingestion run.""" + + document_entity_external_id: UUID + document_file_path: NonEmptyText + raw: DocumentRevisionReferenceV1 + current: DocumentRevisionReferenceV1 | None = None + + @field_validator("document_file_path") + @classmethod + def validate_document_file_path(cls, value: str) -> str: + return _validate_project_relative_path(value) + + @model_validator(mode="after") + def require_raw_extraction_revision(self) -> "DocumentIngestionRunOutputV1": + if self.raw.kind is not DocumentRevisionKind.raw_extraction: + raise ValueError("raw ingestion output must reference a raw_extraction revision") + return self + + +class DocumentIngestionRunFrontmatterV1(_DocumentContractModel): + """Authoritative frontmatter for a ``type: document_ingestion_run`` note.""" + + schema_version: Literal["1"] = "1" + title: NonEmptyText + type: Literal["document_ingestion_run"] = "document_ingestion_run" + schema_ref: Literal["schema/document-ingestion-run"] = Field( + default="schema/document-ingestion-run", + validation_alias=AliasChoices("schema", "schema_ref"), + serialization_alias="schema", + ) + tags: tuple[NonEmptyText, ...] = ("document", "ingestion-run") + permalink: NonEmptyText | None = None + created: datetime | None = None + modified: datetime | None = None + source: DocumentSourceV1 + extraction: DocumentExtractionV1 | None = None + ingestion: DocumentIngestionRunStateV1 + output: DocumentIngestionRunOutputV1 | None = None + failure: DocumentIngestionFailureV1 | None = None + bm_parse_semantics: Literal[False] = False + + @field_validator("created", "modified") + @classmethod + def require_aware_canonical_timestamps(cls, value: datetime | None) -> datetime | None: + if value is None: + return None + return _require_aware_datetime(value, field_name="canonical note timestamp") + + @model_validator(mode="after") + def validate_run_result(self) -> "DocumentIngestionRunFrontmatterV1": + if self.source.checksum != self.ingestion.input_checksum: + raise ValueError("ingestion input_checksum must match the source checksum") + if self.extraction is not None: + _require_deterministic_run_id( + source=self.source, + extraction=self.extraction, + run_id=self.ingestion.run_id, + pipeline_version=self.ingestion.pipeline_version, + prompt_version=self.ingestion.prompt_version, + ) + if self.output is not None: + expected_document_external_id = UUID( + derive_document_note_external_id(self.source.entity_external_id) + ) + if self.output.document_entity_external_id != expected_document_external_id: + raise ValueError("output document identity must match the source entity") + + expected_document_path = derive_document_note_path(self.source.file_path) + if self.output.document_file_path != expected_document_path: + raise ValueError("output document path must be derived from the source path") + + if self.ingestion.stage is DocumentIngestionStage.raw: + if self.extraction is None: + raise ValueError("raw ingestion run requires extraction metadata") + if self.output is None: + raise ValueError("raw ingestion run requires its raw document revision") + if self.output.current is not None: + raise ValueError("raw ingestion run cannot declare a current document revision") + if self.failure is not None: + raise ValueError("raw ingestion run cannot declare a failure") + elif self.ingestion.stage is DocumentIngestionStage.failed: + if self.failure is None: + raise ValueError("failed ingestion run requires structured failure details") + else: + if self.extraction is None: + raise ValueError("successful ingestion run requires extraction metadata") + if self.output is None or self.output.current is None: + raise ValueError("successful ingestion run requires its current document revision") + if self.output.current.kind is DocumentRevisionKind.raw_extraction: + raise ValueError( + "successful ingestion run current revision must be enriched or human-edited" + ) + if self.failure is not None: + raise ValueError("successful ingestion run cannot declare a failure") + return self + + +class DocumentIngestionRunMarkdownV1(_DocumentContractModel): + """Validated ingestion-run frontmatter paired with its Markdown body.""" + + frontmatter: DocumentIngestionRunFrontmatterV1 + body: StrictStr + + +def _parse_agent_semantics(markdown: str) -> "EntityContent": + # Importing the schema package is part of lightweight CLI registration. + # The Markdown parser is only needed when validating actual agent output. + from basic_memory.markdown.entity_parser import parse + + return parse(markdown) + + +class DocumentAgentObservationV1(_DocumentContractModel): + """One bounded observation proposed by an untrusted enrichment agent.""" + + category: Annotated[ + StrictStr, + StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^\[\]()\r\n]+$"), + ] + content: Annotated[ + StrictStr, + StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^\r\n]+$"), + ] + tags: tuple[AgentTag, ...] = () + context: ( + Annotated[ + StrictStr, + StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^\r\n()]+$"), + ] + | None + ) = None + + @model_validator(mode="after") + def require_exact_parser_semantics(self) -> "DocumentAgentObservationV1": + parsed = _parse_agent_semantics(_format_agent_observation(self)) + if parsed.relations or len(parsed.observations) != 1: + raise ValueError("agent observation must produce exactly one observation") + + parsed_observation = parsed.observations[0] + expected_content = self.content + if self.tags: + expected_content += " " + " ".join(f"#{tag}" for tag in self.tags) + if ( + parsed_observation.category != self.category + or parsed_observation.content != expected_content + or tuple(parsed_observation.tags or ()) != self.tags + or parsed_observation.context != self.context + ): + raise ValueError("agent observation fields must match parsed Markdown semantics") + return self + + +class DocumentAgentRelationV1(_DocumentContractModel): + """One bounded relation proposed by an untrusted enrichment agent.""" + + relation_type: Annotated[ + StrictStr, + StringConstraints(pattern=r"^[a-z][a-z0-9_-]*$"), + ] + target: Annotated[ + StrictStr, + StringConstraints(strip_whitespace=True, min_length=1), + ] + context: ( + Annotated[ + StrictStr, + StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^\r\n()]+$"), + ] + | None + ) = None + + @field_validator("target") + @classmethod + def require_single_line_target(cls, value: str) -> str: + if "\n" in value or "\r" in value: + raise ValueError("relation target must fit on one line") + if "[[" in value or "]]" in value: + raise ValueError("relation target cannot contain nested wikilink delimiters") + return value + + @model_validator(mode="after") + def require_exact_parser_semantics(self) -> "DocumentAgentRelationV1": + parsed = _parse_agent_semantics(_format_agent_relation(self)) + if parsed.observations or len(parsed.relations) != 1: + raise ValueError("agent relation must produce exactly one relation") + + parsed_relation = parsed.relations[0] + if ( + parsed_relation.type != self.relation_type + or parsed_relation.target != self.target + or parsed_relation.context != self.context + ): + raise ValueError("agent relation fields must match parsed Markdown semantics") + return self + + +class DocumentAgentOutputV1(_DocumentContractModel): + """Allowed output from a no-tools enrichment agent. + + This model deliberately has no source, extraction, ingestion, checksum, or identity fields. + Trusted code reconstructs that envelope after validating this bounded semantic payload. + """ + + schema_version: Literal["1"] = "1" + title: NonEmptyText + body: StrictStr + tags: tuple[AgentTag, ...] = () + document: DocumentMetadataV1 = Field(default_factory=DocumentMetadataV1) + observations: tuple[DocumentAgentObservationV1, ...] = () + relations: tuple[DocumentAgentRelationV1, ...] = () + + @field_validator("body") + @classmethod + def reject_unstructured_semantics(cls, value: str) -> str: + parsed = _parse_agent_semantics(value) + if parsed.observations or parsed.relations: + raise ValueError( + "agent body cannot contain observations or relations; use structured fields" + ) + return value + + @model_validator(mode="after") + def require_exact_assembled_semantics(self) -> "DocumentAgentOutputV1": + parsed = _parse_agent_semantics(_assemble_agent_body(self)) + expected_observations = [ + _parse_agent_semantics(_format_agent_observation(observation)).observations[0] + for observation in self.observations + ] + expected_relations = [ + _parse_agent_semantics(_format_agent_relation(relation)).relations[0] + for relation in self.relations + ] + if parsed.observations != expected_observations or parsed.relations != expected_relations: + raise ValueError( + "assembled agent Markdown must preserve exactly the declared semantics" + ) + return self + + +_ALLOWED_DOCUMENT_INGESTION_TRANSITIONS: dict[ + DocumentIngestionStage, + frozenset[DocumentIngestionStage], +] = { + DocumentIngestionStage.raw: frozenset( + { + DocumentIngestionStage.ready, + DocumentIngestionStage.needs_review, + DocumentIngestionStage.failed, + } + ), + DocumentIngestionStage.needs_review: frozenset( + {DocumentIngestionStage.ready, DocumentIngestionStage.failed} + ), + DocumentIngestionStage.ready: frozenset(), + DocumentIngestionStage.failed: frozenset(), +} + + +def require_document_ingestion_transition( + current: DocumentIngestionStage, + target: DocumentIngestionStage, +) -> None: + """Fail unless ``target`` is an explicit forward ingestion transition.""" + if target not in _ALLOWED_DOCUMENT_INGESTION_TRANSITIONS[current]: + raise ValueError( + f"invalid document ingestion transition: {current.value} -> {target.value}" + ) + + +def derive_document_note_external_id(source_entity_external_id: str | UUID) -> str: + """Derive one stable note identity from the original file entity identity.""" + source_id = UUID(str(source_entity_external_id)) + return str(uuid5(_DOCUMENT_INGESTION_NAMESPACE, f"document-note:{source_id}")) + + +def derive_document_ingestion_run_id( + *, + source_entity_external_id: str | UUID, + source_checksum: str, + pipeline_version: str, + extractor_engine: str, + extractor_version: str, + extraction_profile: str, + extraction_options_hash: str, + prompt_version: str | None, +) -> str: + """Derive one idempotency identity from every extraction-shaping input.""" + source_id = UUID(str(source_entity_external_id)) + checksum = _validate_sha256_checksum(source_checksum) + options_hash = _validate_sha256_checksum(extraction_options_hash) + normalized_prompt_version = ( + _normalize_run_identity_text(prompt_version, field_name="prompt_version") + if prompt_version is not None + else None + ) + identity = json.dumps( + { + "extraction_options_hash": options_hash, + "extractor_engine": _normalize_run_identity_text( + extractor_engine, + field_name="extractor_engine", + ), + "extractor_version": _normalize_run_identity_text( + extractor_version, + field_name="extractor_version", + ), + "extraction_profile": _normalize_run_identity_text( + extraction_profile, + field_name="extraction_profile", + ), + "pipeline_version": _normalize_run_identity_text( + pipeline_version, + field_name="pipeline_version", + ), + "prompt_version": normalized_prompt_version, + "source_checksum": checksum, + "source_entity_external_id": str(source_id), + }, + sort_keys=True, + separators=(",", ":"), + ) + return str(uuid5(_DOCUMENT_INGESTION_NAMESPACE, identity)) + + +def derive_document_note_path(source_file_path: str) -> str: + """Return the portable sidecar path for one source file (for example ``report.pdf.md``).""" + source_path = _validate_project_relative_path(source_file_path) + return f"{source_path}.md" + + +def derive_document_ingestion_run_path(run_id: str | UUID) -> str: + """Return the stable, path-independent note location for one ingestion run.""" + return f"document-ingestion-runs/{UUID(str(run_id))}.md" + + +def document_markdown_checksum(markdown: str) -> str: + """Return the canonical SHA-256 identity for exact Markdown bytes.""" + return f"sha256:{hashlib.sha256(markdown.encode('utf-8')).hexdigest()}" + + +def assemble_document_markdown(document: DocumentMarkdownV1) -> str: + """Serialize a validated document note into deterministic canonical Markdown.""" + return _assemble_markdown(document.frontmatter, document.body) + + +def parse_document_markdown(markdown: str) -> DocumentMarkdownV1: + """Parse and strictly validate one canonical document note.""" + return DocumentMarkdownV1( + frontmatter=DocumentNoteFrontmatterV1.model_validate(parse_frontmatter(markdown)), + body=_extract_markdown_body(markdown), + ) + + +def assemble_document_ingestion_run_markdown(run: DocumentIngestionRunMarkdownV1) -> str: + """Serialize a validated ingestion-run note into deterministic canonical Markdown.""" + return _assemble_markdown(run.frontmatter, run.body) + + +def parse_document_ingestion_run_markdown(markdown: str) -> DocumentIngestionRunMarkdownV1: + """Parse and strictly validate one canonical ingestion-run note.""" + return DocumentIngestionRunMarkdownV1( + frontmatter=DocumentIngestionRunFrontmatterV1.model_validate(parse_frontmatter(markdown)), + body=_extract_markdown_body(markdown), + ) + + +def enrich_document_markdown( + raw_document: DocumentMarkdownV1, + agent_output: DocumentAgentOutputV1, + target_ingestion: DocumentIngestionV1, +) -> DocumentMarkdownV1: + """Build an enriched note while preserving the trusted raw provenance envelope.""" + raw_frontmatter = raw_document.frontmatter + if raw_frontmatter.ingestion.stage is not DocumentIngestionStage.raw: + raise ValueError("agent enrichment requires a raw document") + require_document_ingestion_transition( + raw_frontmatter.ingestion.stage, + target_ingestion.stage, + ) + if target_ingestion.stage not in { + DocumentIngestionStage.ready, + DocumentIngestionStage.needs_review, + }: + raise ValueError("agent enrichment can only produce ready or needs_review documents") + + expected_base_checksum = document_markdown_checksum(assemble_document_markdown(raw_document)) + if target_ingestion.base_checksum != expected_base_checksum: + raise ValueError("target ingestion base_checksum does not match the raw document") + if ( + target_ingestion.run_id != raw_frontmatter.ingestion.run_id + or target_ingestion.input_checksum != raw_frontmatter.ingestion.input_checksum + or target_ingestion.pipeline_version != raw_frontmatter.ingestion.pipeline_version + or target_ingestion.prompt_version != raw_frontmatter.ingestion.prompt_version + ): + raise ValueError("enrichment cannot replace trusted ingestion identity or version fields") + + tags = tuple(dict.fromkeys((*raw_frontmatter.tags, *agent_output.tags))) + return DocumentMarkdownV1( + frontmatter=DocumentNoteFrontmatterV1( + title=agent_output.title, + tags=tags, + permalink=raw_frontmatter.permalink, + created=raw_frontmatter.created, + modified=raw_frontmatter.modified, + source=raw_frontmatter.source, + extraction=raw_frontmatter.extraction, + ingestion=target_ingestion, + document=agent_output.document, + bm_parse_semantics=True, + ), + body=_assemble_agent_body(agent_output), + ) + + +def _assemble_agent_body(agent_output: DocumentAgentOutputV1) -> str: + sections: list[str] = [] + if normalized_body := _normalize_markdown_body(agent_output.body): + sections.append(normalized_body.rstrip("\n")) + if agent_output.observations: + observations = [ + _format_agent_observation(observation) for observation in agent_output.observations + ] + sections.append("## Observations\n\n" + "\n".join(observations)) + if agent_output.relations: + relations = [_format_agent_relation(relation) for relation in agent_output.relations] + sections.append("## Relations\n\n" + "\n".join(relations)) + return "\n\n".join(sections) + ("\n" if sections else "") + + +def _format_agent_observation(observation: DocumentAgentObservationV1) -> str: + line = f"- [{observation.category}] {observation.content}" + if observation.tags: + line += " " + " ".join(f"#{tag}" for tag in observation.tags) + if observation.context: + line += f" ({observation.context})" + return line + + +def _format_agent_relation(relation: DocumentAgentRelationV1) -> str: + line = f"- {relation.relation_type} [[{relation.target}]]" + if relation.context: + line += f" ({relation.context})" + return line + + +def _assemble_markdown(frontmatter: _DocumentContractModel, body: str) -> str: + metadata = frontmatter.model_dump(mode="json", exclude_none=True, by_alias=True) + return dump_frontmatter(Post(_normalize_markdown_body(body), **metadata)) + + +def _extract_markdown_body(markdown: str) -> str: + body = remove_frontmatter(markdown, strip=False).lstrip("\r\n") + return _normalize_markdown_body(body) + + +def _normalize_markdown_body(body: str) -> str: + normalized = body.replace("\r\n", "\n").replace("\r", "\n").strip("\n") + return f"{normalized}\n" if normalized else "" + + +def _require_aware_datetime(value: datetime, *, field_name: str) -> datetime: + if value.utcoffset() is None: + raise ValueError(f"{field_name} must include a timezone") + return value + + +def _validate_sha256_checksum(value: str) -> str: + if _SHA256_CHECKSUM_PATTERN.fullmatch(value) is None: + raise ValueError("checksum must use canonical sha256: form") + return value + + +def _require_deterministic_run_id( + *, + source: DocumentSourceV1, + extraction: DocumentExtractionV1, + run_id: UUID, + pipeline_version: str, + prompt_version: str | None, +) -> None: + expected_run_id = UUID( + derive_document_ingestion_run_id( + source_entity_external_id=source.entity_external_id, + source_checksum=source.checksum, + pipeline_version=pipeline_version, + extractor_engine=extraction.engine, + extractor_version=extraction.engine_version, + extraction_profile=extraction.profile, + extraction_options_hash=extraction.options_hash, + prompt_version=prompt_version, + ) + ) + if run_id != expected_run_id: + raise ValueError( + "ingestion run_id must match its deterministic source and pipeline identity" + ) + + +def _normalize_run_identity_text(value: str, *, field_name: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError(f"{field_name} cannot be blank") + return normalized + + +def _validate_project_relative_path(value: str) -> str: + if not value or "\\" in value or "\x00" in value: + raise ValueError("path must be a non-empty POSIX project-relative path") + path = PurePosixPath(value) + if ( + not path.parts + or path.is_absolute() + or ".." in path.parts + or path.as_posix() != value + or value.endswith("/") + ): + raise ValueError("path must be a canonical POSIX project-relative path") + return value diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index f6159da42..553142bb1 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -332,6 +332,27 @@ async def test_parse_invalid_canonical_timestamp_fails_for_field( ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "semantic_setting", + [ + "bm_parse_semantics:\n - false", + "bm_parse_semantics:\n enabled: false", + ], +) +async def test_non_scalar_semantic_setting_does_not_crash( + entity_parser, + semantic_setting: str, +) -> None: + entity = await entity_parser.parse_markdown_content( + Path("non-scalar-semantics.md"), + f"---\n{semantic_setting}\n---\n- [note] Still parsed\n- references [[Target]]", + ) + + assert [observation.content for observation in entity.observations] == ["Still parsed"] + assert [relation.target for relation in entity.relations] == ["Target"] + + # @pytest.mark.asyncio # async def test_parse_file_invalid_yaml(test_config, entity_parser): # """Test parsing file with invalid YAML frontmatter.""" diff --git a/tests/schemas/test_document.py b/tests/schemas/test_document.py new file mode 100644 index 000000000..2a594d2dd --- /dev/null +++ b/tests/schemas/test_document.py @@ -0,0 +1,961 @@ +"""Tests for the portable document-ingestion contracts.""" + +from datetime import UTC, datetime +from uuid import UUID + +import pytest +from pydantic import ValidationError + +from basic_memory.markdown.entity_parser import EntityParser +from basic_memory.schemas.document import ( + DocumentAgentObservationV1, + DocumentAgentOutputV1, + DocumentAgentRelationV1, + DocumentExtractionStatus, + DocumentExtractionV1, + DocumentIngestionFailureV1, + DocumentIngestionRunFrontmatterV1, + DocumentIngestionRunMarkdownV1, + DocumentIngestionRunOutputV1, + DocumentIngestionRunStateV1, + DocumentIngestionStage, + DocumentIngestionV1, + DocumentMarkdownV1, + DocumentMetadataV1, + DocumentNoteFrontmatterV1, + DocumentRevisionKind, + DocumentRevisionReferenceV1, + DocumentSourceV1, + assemble_document_ingestion_run_markdown, + assemble_document_markdown, + derive_document_ingestion_run_id, + derive_document_ingestion_run_path, + derive_document_note_external_id, + derive_document_note_path, + document_markdown_checksum, + enrich_document_markdown, + parse_document_ingestion_run_markdown, + parse_document_markdown, + require_document_ingestion_transition, +) + + +SOURCE_ENTITY_ID = UUID("11111111-1111-1111-1111-111111111111") +SOURCE_CHECKSUM = f"sha256:{'a' * 64}" +EXTRACTION_OPTIONS_HASH = f"sha256:{'d' * 64}" +EXTRACTED_AT = datetime(2026, 8, 1, 1, 2, 3, tzinfo=UTC) +CANONICAL_CREATED = datetime(2026, 8, 1, 1, 3, tzinfo=UTC) +CANONICAL_MODIFIED = datetime(2026, 8, 1, 1, 4, tzinfo=UTC) +RUN_ID = UUID( + derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum=SOURCE_CHECKSUM, + pipeline_version="bm-document-ingest-v1", + extractor_engine="firecrawl/pdf-inspector", + extractor_version="0.2.6", + extraction_profile="pdf-inspector-v1", + extraction_options_hash=EXTRACTION_OPTIONS_HASH, + prompt_version="document-normalize-v1", + ) +) + + +def source() -> DocumentSourceV1: + return DocumentSourceV1( + media_type="application/pdf", + entity_external_id=SOURCE_ENTITY_ID, + file_path="docs/report.pdf", + checksum=SOURCE_CHECKSUM, + size_bytes=1024, + storage_version_id="tigris-source-v1", + storage_etag='"source-etag-v1"', + ) + + +def extraction() -> DocumentExtractionV1: + return DocumentExtractionV1( + engine="firecrawl/pdf-inspector", + engine_version="0.2.6", + profile="pdf-inspector-v1", + options_hash=EXTRACTION_OPTIONS_HASH, + classification="mixed", + status=DocumentExtractionStatus.needs_ocr, + extracted_at=EXTRACTED_AT, + duration_ms=12, + page_count=3, + extracted_page_count=2, + requires_ocr=True, + ocr_page_count=1, + pages_needing_ocr=(3,), + confidence=0.91, + has_tables=True, + has_columns=False, + ) + + +def raw_document() -> DocumentMarkdownV1: + return DocumentMarkdownV1( + frontmatter=DocumentNoteFrontmatterV1( + title="Report", + tags=("document", "pdf", "imported"), + permalink="docs/report-pdf", + created=CANONICAL_CREATED, + modified=CANONICAL_MODIFIED, + source=source(), + extraction=extraction(), + ingestion=DocumentIngestionV1( + stage=DocumentIngestionStage.raw, + pipeline_version="bm-document-ingest-v1", + prompt_version="document-normalize-v1", + run_id=RUN_ID, + input_checksum=SOURCE_CHECKSUM, + ), + bm_parse_semantics=False, + ), + body="# Extracted report\r\n\r\n- [accidental] not an observation\r\n\r\n[[Not a relation]]\r\n", + ) + + +def raw_revision() -> DocumentRevisionReferenceV1: + return DocumentRevisionReferenceV1( + kind=DocumentRevisionKind.raw_extraction, + checksum=document_markdown_checksum(assemble_document_markdown(raw_document())), + storage_version_id="tigris-v1", + created_at=EXTRACTED_AT, + ) + + +def test_document_markdown_round_trip_is_canonical() -> None: + document = raw_document() + + first = assemble_document_markdown(document) + second = assemble_document_markdown(document) + + assert first == second + assert first.startswith("---\nschema_version: '1'\ntitle: Report\ntype: document\n") + assert first.endswith("[[Not a relation]]\n") + assert "\r" not in first + assert parse_document_markdown(first) == DocumentMarkdownV1( + frontmatter=document.frontmatter, + body="# Extracted report\n\n- [accidental] not an observation\n\n[[Not a relation]]\n", + ) + assert ( + DocumentNoteFrontmatterV1.model_validate(document.frontmatter.model_dump()) + == document.frontmatter + ) + + +def test_document_contract_rejects_invalid_nested_provenance() -> None: + data = source().model_dump(mode="json") + data["extra"] = "not trusted" + + with pytest.raises(ValidationError, match="extra"): + DocumentSourceV1.model_validate(data) + + with pytest.raises(ValidationError, match="checksum"): + DocumentSourceV1.model_validate({**source().model_dump(), "checksum": "A" * 64}) + + with pytest.raises(ValidationError, match="pages_needing_ocr"): + DocumentExtractionV1.model_validate( + { + **extraction().model_dump(), + "ocr_page_count": 2, + } + ) + + with pytest.raises(ValidationError, match="timezone"): + DocumentExtractionV1.model_validate( + { + **extraction().model_dump(), + "extracted_at": datetime(2026, 8, 1, 1, 2, 3), + } + ) + + +@pytest.mark.parametrize("page", [True, 1.0, "1"]) +def test_extraction_rejects_coerced_ocr_page_numbers(page: object) -> None: + with pytest.raises(ValidationError, match="pages_needing_ocr"): + DocumentExtractionV1.model_validate( + { + **extraction().model_dump(), + "pages_needing_ocr": (page,), + } + ) + + +@pytest.mark.parametrize( + ("updates", "message"), + [ + ({"extracted_page_count": 4}, "extracted_page_count"), + ({"pages_needing_ocr": (3, 2), "ocr_page_count": 2}, "unique and sorted"), + ({"pages_needing_ocr": (4,)}, "one-based page numbers"), + ({"requires_ocr": False}, "requires_ocr"), + ( + { + "status": DocumentExtractionStatus.needs_ocr, + "requires_ocr": False, + "ocr_page_count": 0, + "pages_needing_ocr": (), + }, + "needs_ocr extraction status", + ), + ( + { + "status": DocumentExtractionStatus.complete, + "requires_ocr": False, + "ocr_page_count": 0, + "pages_needing_ocr": (), + }, + "complete extraction", + ), + ], +) +def test_extraction_page_invariants_fail_fast( + updates: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValidationError, match=message): + DocumentExtractionV1.model_validate({**extraction().model_dump(), **updates}) + + +def test_document_frontmatter_requires_consistent_trusted_metadata() -> None: + raw = raw_document() + + with pytest.raises(ValidationError, match="raw ingestion stage"): + DocumentIngestionV1.model_validate( + { + **raw.frontmatter.ingestion.model_dump(), + "base_checksum": f"sha256:{'b' * 64}", + } + ) + with pytest.raises(ValidationError, match="non-raw ingestion stages"): + DocumentIngestionV1( + stage=DocumentIngestionStage.ready, + pipeline_version="pipeline-v1", + run_id=RUN_ID, + input_checksum=SOURCE_CHECKSUM, + ) + with pytest.raises(ValidationError, match="authors"): + DocumentMetadataV1(authors=("Ada", "Ada")) + + frontmatter_data = raw.frontmatter.model_dump(mode="json", by_alias=True) + with pytest.raises(ValidationError, match="tags"): + DocumentNoteFrontmatterV1.model_validate( + {**frontmatter_data, "tags": ["document", "document"]} + ) + with pytest.raises(ValidationError, match="input_checksum"): + DocumentNoteFrontmatterV1.model_validate( + { + **frontmatter_data, + "ingestion": { + **frontmatter_data["ingestion"], + "input_checksum": f"sha256:{'b' * 64}", + }, + } + ) + with pytest.raises(ValidationError, match="run_id"): + DocumentNoteFrontmatterV1.model_validate( + { + **frontmatter_data, + "ingestion": { + **frontmatter_data["ingestion"], + "run_id": "22222222-2222-2222-2222-222222222222", + }, + } + ) + with pytest.raises(ValidationError, match="bm_parse_semantics"): + DocumentNoteFrontmatterV1.model_validate({**frontmatter_data, "bm_parse_semantics": True}) + without_canonical_timestamps = DocumentNoteFrontmatterV1.model_validate( + {**frontmatter_data, "created": None, "modified": None} + ) + assert without_canonical_timestamps.created is None + + with pytest.raises(ValidationError, match="canonical note timestamp"): + DocumentNoteFrontmatterV1.model_validate( + {**frontmatter_data, "created": datetime(2026, 8, 1)} + ) + + +def test_ingestion_stages_require_explicit_forward_transitions() -> None: + require_document_ingestion_transition( + DocumentIngestionStage.raw, + DocumentIngestionStage.needs_review, + ) + require_document_ingestion_transition( + DocumentIngestionStage.needs_review, + DocumentIngestionStage.ready, + ) + + with pytest.raises(ValueError, match="raw -> raw"): + require_document_ingestion_transition( + DocumentIngestionStage.raw, + DocumentIngestionStage.raw, + ) + with pytest.raises(ValueError, match="ready -> failed"): + require_document_ingestion_transition( + DocumentIngestionStage.ready, + DocumentIngestionStage.failed, + ) + + +def test_document_identities_and_paths_are_deterministic() -> None: + assert derive_document_note_external_id(SOURCE_ENTITY_ID) == derive_document_note_external_id( + str(SOURCE_ENTITY_ID) + ) + assert derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum=SOURCE_CHECKSUM, + pipeline_version="bm-document-ingest-v1", + extractor_engine="firecrawl/pdf-inspector", + extractor_version="0.2.6", + extraction_profile="pdf-inspector-v1", + extraction_options_hash=EXTRACTION_OPTIONS_HASH, + prompt_version="document-normalize-v1", + ) == str(RUN_ID) + assert derive_document_note_path("docs/report.pdf") == "docs/report.pdf.md" + assert derive_document_ingestion_run_path(RUN_ID) == f"document-ingestion-runs/{RUN_ID}.md" + + with pytest.raises(ValueError, match="project-relative"): + derive_document_note_path("../report.pdf") + with pytest.raises(ValueError, match="project-relative"): + derive_document_note_path(".") + with pytest.raises(ValueError, match="canonical sha256"): + derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum="a" * 64, + pipeline_version="pipeline-v1", + extractor_engine="extractor", + extractor_version="1", + extraction_profile="profile-v1", + extraction_options_hash=EXTRACTION_OPTIONS_HASH, + prompt_version=None, + ) + with pytest.raises(ValueError, match="pipeline_version"): + derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum=SOURCE_CHECKSUM, + pipeline_version=" ", + extractor_engine="extractor", + extractor_version="1", + extraction_profile="profile-v1", + extraction_options_hash=EXTRACTION_OPTIONS_HASH, + prompt_version=None, + ) + + +@pytest.mark.parametrize( + ( + "extractor_engine", + "extractor_version", + "extraction_profile", + "extraction_options_hash", + "prompt_version", + ), + [ + ( + "other-extractor", + "0.2.6", + "pdf-inspector-v1", + EXTRACTION_OPTIONS_HASH, + "document-normalize-v1", + ), + ( + "firecrawl/pdf-inspector", + "0.2.7", + "pdf-inspector-v1", + EXTRACTION_OPTIONS_HASH, + "document-normalize-v1", + ), + ( + "firecrawl/pdf-inspector", + "0.2.6", + "pdf-inspector-v2", + EXTRACTION_OPTIONS_HASH, + "document-normalize-v1", + ), + ( + "firecrawl/pdf-inspector", + "0.2.6", + "pdf-inspector-v1", + f"sha256:{'e' * 64}", + "document-normalize-v1", + ), + ( + "firecrawl/pdf-inspector", + "0.2.6", + "pdf-inspector-v1", + EXTRACTION_OPTIONS_HASH, + "document-normalize-v2", + ), + ( + "firecrawl/pdf-inspector", + "0.2.6", + "pdf-inspector-v1", + EXTRACTION_OPTIONS_HASH, + None, + ), + ], +) +def test_ingestion_run_identity_includes_every_extraction_shaping_input( + extractor_engine: str, + extractor_version: str, + extraction_profile: str, + extraction_options_hash: str, + prompt_version: str | None, +) -> None: + assert derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum=SOURCE_CHECKSUM, + pipeline_version="bm-document-ingest-v1", + extractor_engine=extractor_engine, + extractor_version=extractor_version, + extraction_profile=extraction_profile, + extraction_options_hash=extraction_options_hash, + prompt_version=prompt_version, + ) != str(RUN_ID) + + +@pytest.mark.parametrize( + ( + "pipeline_version", + "extractor_engine", + "extractor_version", + "extraction_profile", + "prompt_version", + "message", + ), + [ + (" ", "extractor", "1", "profile", "prompt", "pipeline_version"), + ("pipeline", " ", "1", "profile", "prompt", "extractor_engine"), + ("pipeline", "extractor", " ", "profile", "prompt", "extractor_version"), + ("pipeline", "extractor", "1", " ", "prompt", "extraction_profile"), + ("pipeline", "extractor", "1", "profile", " ", "prompt_version"), + ], +) +def test_ingestion_run_identity_rejects_blank_versions( + pipeline_version: str, + extractor_engine: str, + extractor_version: str, + extraction_profile: str, + prompt_version: str, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum=SOURCE_CHECKSUM, + pipeline_version=pipeline_version, + extractor_engine=extractor_engine, + extractor_version=extractor_version, + extraction_profile=extraction_profile, + extraction_options_hash=EXTRACTION_OPTIONS_HASH, + prompt_version=prompt_version, + ) + + +def test_ingestion_run_identity_requires_canonical_options_hash() -> None: + with pytest.raises(ValueError, match="canonical sha256"): + derive_document_ingestion_run_id( + source_entity_external_id=SOURCE_ENTITY_ID, + source_checksum=SOURCE_CHECKSUM, + pipeline_version="bm-document-ingest-v1", + extractor_engine="firecrawl/pdf-inspector", + extractor_version="0.2.6", + extraction_profile="pdf-inspector-v1", + extraction_options_hash="d" * 64, + prompt_version="document-normalize-v1", + ) + + +def test_agent_enrichment_preserves_provenance_and_requires_raw_checksum() -> None: + raw = raw_document() + raw_checksum = document_markdown_checksum(assemble_document_markdown(raw)) + agent_output = DocumentAgentOutputV1( + title="Quarterly Report", + body="# Quarterly Report\n\nA normalized summary.", + tags=("quarterly",), + document=DocumentMetadataV1( + kind="report", + language="en", + authors=("Ada Example",), + ), + observations=( + DocumentAgentObservationV1( + category="summary", + content="Revenue increased.", + tags=("finance",), + context="reported result", + ), + ), + relations=( + DocumentAgentRelationV1( + relation_type="references", + target="Finance plan", + context="supporting plan", + ), + ), + ) + target_ingestion = DocumentIngestionV1( + stage=DocumentIngestionStage.ready, + pipeline_version=raw.frontmatter.ingestion.pipeline_version, + prompt_version=raw.frontmatter.ingestion.prompt_version, + run_id=raw.frontmatter.ingestion.run_id, + input_checksum=raw.frontmatter.ingestion.input_checksum, + base_checksum=raw_checksum, + ) + + enriched = enrich_document_markdown(raw, agent_output, target_ingestion) + + assert enriched.frontmatter.source == raw.frontmatter.source + assert enriched.frontmatter.extraction == raw.frontmatter.extraction + assert enriched.frontmatter.ingestion == target_ingestion + assert enriched.frontmatter.permalink == raw.frontmatter.permalink + assert enriched.frontmatter.created == raw.frontmatter.created + assert enriched.frontmatter.modified == raw.frontmatter.modified + assert enriched.frontmatter.tags == ("document", "pdf", "imported", "quarterly") + assert enriched.frontmatter.bm_parse_semantics is True + assert "- [summary] Revenue increased. #finance (reported result)" in enriched.body + assert "- references [[Finance plan]] (supporting plan)" in enriched.body + + with pytest.raises(ValidationError, match="source"): + DocumentAgentOutputV1.model_validate( + { + **agent_output.model_dump(mode="json"), + "source": source().model_dump(mode="json"), + } + ) + + wrong_base = target_ingestion.model_copy(update={"base_checksum": f"sha256:{'b' * 64}"}) + with pytest.raises(ValueError, match="base_checksum"): + enrich_document_markdown(raw, agent_output, wrong_base) + + failed_target = target_ingestion.model_copy(update={"stage": DocumentIngestionStage.failed}) + with pytest.raises(ValueError, match="only produce ready or needs_review"): + enrich_document_markdown(raw, agent_output, failed_target) + + different_run = target_ingestion.model_copy( + update={"run_id": UUID("22222222-2222-2222-2222-222222222222")} + ) + with pytest.raises(ValueError, match="trusted ingestion identity"): + enrich_document_markdown(raw, agent_output, different_run) + + needs_review_target = target_ingestion.model_copy( + update={"stage": DocumentIngestionStage.needs_review} + ) + needs_review = enrich_document_markdown(raw, agent_output, needs_review_target) + reviewed_checksum = document_markdown_checksum(assemble_document_markdown(needs_review)) + with pytest.raises(ValueError, match="requires a raw document"): + enrich_document_markdown( + needs_review, + agent_output, + target_ingestion.model_copy(update={"base_checksum": reviewed_checksum}), + ) + + +@pytest.mark.parametrize( + "body", + [ + "- [injected] Unstructured observation", + "A summary with [[Injected relation]].", + "A summary with #injected-tag", + ], +) +def test_agent_body_rejects_unstructured_semantics(body: str) -> None: + with pytest.raises(ValidationError, match="structured fields"): + DocumentAgentOutputV1(title="Unsafe output", body=body) + + +@pytest.mark.parametrize("body", ["Summary.\n\n```", "Summary.\n\n