Skip to content

feat(core): add portable document ingestion contracts #1178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
phernandez wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(core): add portable document ingestion contracts #1178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/basic_memory/markdown/entity_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,9 +322,18 @@ async def parse_markdown_content(
if tags:
metadata["tags"] = tags

# Parse content for observations and relations
# Raw extraction text is useful for retrieval but is untrusted semantic input.
# The ingestion contract sets this explicit opt-out so arbitrary PDF syntax cannot
# mint observations or relations before the bounded enrichment pass.
entity_frontmatter = EntityFrontmatter(metadata=metadata)
entity_content = parse(post.content)
semantic_setting = metadata.get("bm_parse_semantics")
parse_semantics = not (
semantic_setting is False
or (isinstance(semantic_setting, str) and semantic_setting.lower() == "false")
)
entity_content = (
parse(post.content) if parse_semantics else EntityContent(content=post.content)
)

# Canonical frontmatter timestamps describe note semantics. File times are
# only compatibility fallbacks for notes that do not declare them.
Expand Down
4 changes: 3 additions & 1 deletion src/basic_memory/runtime/note_object_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@
# collaboration_relay = a relay service persisting a live collaboration
# document with its service credential (issue #1445); the webhook-canonical
# note.updated event echoes this source as the write's actor origin.
# document_ingestion = a hosted worker accepting a deterministic document
# extraction or ingestion-run note through the canonical note mutation path.
VALID_NOTE_OBJECT_SOURCES: frozenset[RuntimeNoteChangeSource] = frozenset(
{"api", "collaboration_relay", "mcp", "s3_webhook", "web_v2"}
{"api", "collaboration_relay", "document_ingestion", "mcp", "s3_webhook", "web_v2"}
)
# Named because the accepted-note write path special-cases relay writes: the
# relay superseding its own prior write is never a real conflict (#1589).
Expand Down
62 changes: 62 additions & 0 deletions src/basic_memory/schemas/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,38 @@
DirectoryNode,
)

from basic_memory.schemas.document import (
DocumentAgentObservationV1,
DocumentAgentOutputV1,
DocumentAgentRelationV1,
DocumentExtractionStatus,
DocumentExtractionV1,
DocumentIngestionFailureV1,
DocumentIngestionRunFrontmatterV1,
DocumentIngestionRunMarkdownV1,
DocumentIngestionRunOutputV1,
DocumentIngestionRunStateV1,
DocumentIngestionStage,
DocumentIngestionV1,
DocumentMarkdownV1,
DocumentMetadataV1,
DocumentNoteFrontmatterV1,
DocumentRevisionKind,
DocumentRevisionReferenceV1,
DocumentSourceV1,
assemble_document_ingestion_run_markdown,
assemble_document_markdown,
derive_document_ingestion_run_id,
derive_document_ingestion_run_path,
derive_document_note_external_id,
derive_document_note_path,
document_markdown_checksum,
enrich_document_markdown,
parse_document_ingestion_run_markdown,
parse_document_markdown,
require_document_ingestion_transition,
)

# For convenient imports, export all models
__all__ = [
# Base
Expand Down Expand Up @@ -88,4 +120,34 @@
"ProjectIndexStatusResponse",
# Directory
"DirectoryNode",
# Document ingestion
"DocumentAgentObservationV1",
"DocumentAgentOutputV1",
"DocumentAgentRelationV1",
"DocumentExtractionStatus",
"DocumentExtractionV1",
"DocumentIngestionFailureV1",
"DocumentIngestionRunFrontmatterV1",
"DocumentIngestionRunMarkdownV1",
"DocumentIngestionRunOutputV1",
"DocumentIngestionRunStateV1",
"DocumentIngestionStage",
"DocumentIngestionV1",
"DocumentMarkdownV1",
"DocumentMetadataV1",
"DocumentNoteFrontmatterV1",
"DocumentRevisionKind",
"DocumentRevisionReferenceV1",
"DocumentSourceV1",
"assemble_document_ingestion_run_markdown",
"assemble_document_markdown",
"derive_document_ingestion_run_id",
"derive_document_ingestion_run_path",
"derive_document_note_external_id",
"derive_document_note_path",
"document_markdown_checksum",
"enrich_document_markdown",
"parse_document_ingestion_run_markdown",
"parse_document_markdown",
"require_document_ingestion_transition",
]
Loading
Loading