diff --git a/Cargo.lock b/Cargo.lock index e989458..62b7983 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1712,6 +1712,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "tempfile", "thiserror 1.0.69", "uuid", @@ -1723,6 +1724,7 @@ version = "0.12.0" dependencies = [ "rusqlite", "serde_json", + "sha2", "tempfile", "tree-ring-memory-core", ] diff --git a/Cargo.toml b/Cargo.toml index 2ab29b3..ee05dc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,13 +15,14 @@ repository = "https://github.com/TerminallyLazy/Tree-Ring-Memory" [workspace.dependencies] chrono = { version = "0.4", features = ["serde", "clock"] } -clap = { version = "4", features = ["derive"] } +clap = { version = "4", features = ["derive", "env"] } libc = "0.2" once_cell = "1" regex = "1" rusqlite = { version = "0.32", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" tempfile = "3" thiserror = "1" uuid = { version = "1", features = ["v4"] } diff --git a/README.md b/README.md index 389f6aa..f7a9d1d 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ Tree Ring Memory is in protocol-preview status. Current launch links: project truths. - Rust-native import/export, audit, consolidation, maintenance, DOX/Revolve adapters, harness discovery, and terminal UI. +- Same-host multi-agent correlation, scoped recall filters, and idempotent + worker writes through the public CLI. - Evidence artifacts for install size, recall speed, harness readiness, and recall quality. - Privacy defaults that block secret-like memory, hide sensitive details, and @@ -236,6 +238,70 @@ Command ownership is Rust-native: - `export`, `import`, `audit`, `consolidate`, and `maintain` are local maintenance surfaces over the same SQLite store. - `welcome` and `tui` are the terminal onboarding and operator-console surfaces. +## Same-Host Multi-Agent Workflow + +A coordinator can fan work out to multiple local CLI processes that share one +Tree Ring root. Give every worker a unique agent profile and operation ID while +sharing the workflow and session: + +```bash +tree-ring --root .tree-ring remember "Storage worker validated WAL behavior." \ + --event-type lesson \ + --scope agent \ + --project example-service \ + --agent-profile worker-storage \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --operation-id validate-storage-v1 \ + --source-ref runs/release-readiness/worker-storage.json +``` + +At fan-in, omit the agent-profile filter so the coordinator sees every +agent-partitioned result: + +```bash +tree-ring --root .tree-ring --json recall "release readiness" \ + --project example-service \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --scope agent \ + --limit 64 +``` + +When writing, `scope=agent` requires `agent_profile`, `scope=workflow` requires +`workflow_id`, and `scope=session` requires `session_id`. A coordinator's +aggregate recall may intentionally omit the agent-profile filter. Project and +global scopes remain shared. These are routing and consolidation partitions, +not ACLs; any process with filesystem access to the store can perform +unfiltered recall. +Pre-0.12 private-scope records that lack the now-required identity are migrated +to a deterministic, per-record `legacy-*` partition and marked for review +instead of being widened into shared scope or becoming unexportable. + +`TREE_RING_AGENT_PROFILE`, `TREE_RING_WORKFLOW_ID`, and +`TREE_RING_SESSION_ID` can supply the matching flag defaults. `operation_id` +provides write idempotency inside the `(project, workflow_id, agent_profile)` +namespace: an exact retry returns the existing memory ID, while a different +payload using the same key fails nonzero. Session ID is retained context but is +not part of that idempotency namespace. Replacing an active row preserves its +prior operation namespace as a one-way claim. Redaction also keeps a memory-ID +tombstone, so replacement import cannot restore the payload by omitting the +operation ID; only explicit hard deletion releases those claims. + +The shared-root contract is limited to concurrent processes on one host using a +local filesystem. Tree Ring does not claim distributed locking, cross-host +SQLite coordination, or safe database sharing over NFS/network filesystems. +For work spanning hosts, keep per-host roots and use an explicit, +evidence-preserving fan-in. + +The bounded acceptance test at +`crates/tree-ring-memory-cli/tests/multi_agent_acceptance.rs` holds a real +SQLite write lock, starts eight real CLI workers, verifies they wait and then +complete, exercises each recall filter and operation conflict behavior, and +checks exact row/FTS parity through `tree-ring --json maintain`. This is +same-host evidence, not sustained-load, crash-recovery, fairness, or distributed +storage certification. + ## Evidence Loop The Revolve-inspired loop is exposed through `tree-ring evidence`. It records @@ -422,6 +488,7 @@ cargo run -p tree-ring-memory-cli -- maintain --help cargo run -p tree-ring-memory-cli -- dox sync --help cargo run -p tree-ring-memory-cli -- revolve sync --help cargo run -p tree-ring-memory-cli -- integrations scan --help +cargo test -p tree-ring-memory-cli --test multi_agent_acceptance cargo run --release -p tree-ring-memory-sqlite --example performance_smoke -- 1000 sh scripts/certify-tree-ring.sh sh scripts/package-release.sh diff --git a/crates/tree-ring-memory-cli/src/actions/lifecycle.rs b/crates/tree-ring-memory-cli/src/actions/lifecycle.rs index 0a33019..b9e102d 100644 --- a/crates/tree-ring-memory-cli/src/actions/lifecycle.rs +++ b/crates/tree-ring-memory-cli/src/actions/lifecycle.rs @@ -13,6 +13,9 @@ pub struct ConsolidateActionRequest { pub period_type: String, pub period_key: Option, pub project: Option, + pub agent_profile: Option, + pub workflow_id: Option, + pub session_id: Option, pub dry_run: bool, pub force: bool, } @@ -34,6 +37,9 @@ pub fn consolidation_request( .map_err(|err| err.to_string())?, period_key: request.period_key, project: request.project, + agent_profile: request.agent_profile, + workflow_id: request.workflow_id, + session_id: request.session_id, dry_run: request.dry_run, force: request.force, }) diff --git a/crates/tree-ring-memory-cli/src/actions/recall.rs b/crates/tree-ring-memory-cli/src/actions/recall.rs index bd2a6ec..7cc1a8e 100644 --- a/crates/tree-ring-memory-cli/src/actions/recall.rs +++ b/crates/tree-ring-memory-cli/src/actions/recall.rs @@ -1,4 +1,4 @@ -use tree_ring_memory_sqlite::{MemoryRetriever, RecallResult, SQLiteMemoryStore}; +use tree_ring_memory_sqlite::{MemoryRetriever, RecallOptions, RecallResult, SQLiteMemoryStore}; use super::ActionResult; @@ -6,6 +6,10 @@ use super::ActionResult; pub struct RecallRequest { pub query: String, pub project: Option, + pub agent_profile: Option, + pub workflow_id: Option, + pub session_id: Option, + pub scope: Option, pub limit: usize, pub include_sensitive: bool, pub include_superseded: bool, @@ -19,17 +23,21 @@ pub struct RecallReport { pub fn recall(store: &SQLiteMemoryStore, request: RecallRequest) -> ActionResult { let results = MemoryRetriever::new(store) - .recall( + .recall_with_options( &request.query, - request.project.as_deref(), - None, - None, - None, - None, - request.include_sensitive, - request.include_superseded, - request.limit, - request.explain, + &RecallOptions { + project: request.project.as_deref(), + agent_profile: request.agent_profile.as_deref(), + workflow_id: request.workflow_id.as_deref(), + session_id: request.session_id.as_deref(), + scope: request.scope.as_deref(), + rings: None, + event_types: None, + include_sensitive: request.include_sensitive, + include_superseded: request.include_superseded, + limit: request.limit, + explain_ranking: request.explain, + }, ) .map_err(|err| err.to_string())?; Ok(RecallReport { results }) @@ -54,6 +62,10 @@ mod tests { RecallRequest { query: "shared recall".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + scope: None, limit: 8, include_sensitive: false, include_superseded: false, @@ -66,4 +78,45 @@ mod tests { assert_eq!(report.results[0].memory.id, event.id); assert!(report.results[0].ranking.contains_key("textual_match")); } + + #[test] + fn recall_action_filters_multi_agent_context_before_ranking() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + for (agent, workflow, session) in [ + ("researcher", "fanout-7", "attempt-1"), + ("reviewer", "fanout-7", "attempt-1"), + ("researcher", "fanout-8", "attempt-2"), + ] { + let mut event = MemoryEvent::new("Shared phrase from worker.", "lesson").unwrap(); + event.scope = "agent".to_string(); + event.agent_profile = Some(agent.to_string()); + event.workflow_id = Some(workflow.to_string()); + event.session_id = Some(session.to_string()); + store.put(&event).unwrap(); + } + + let report = recall( + &store, + RecallRequest { + query: "shared phrase worker".to_string(), + project: None, + agent_profile: Some("researcher".to_string()), + workflow_id: Some("fanout-7".to_string()), + session_id: Some("attempt-1".to_string()), + scope: Some("agent".to_string()), + limit: 8, + include_sensitive: false, + include_superseded: false, + explain: false, + }, + ) + .unwrap(); + + assert_eq!(report.results.len(), 1); + assert_eq!( + report.results[0].memory.agent_profile.as_deref(), + Some("researcher") + ); + } } diff --git a/crates/tree-ring-memory-cli/src/actions/remember.rs b/crates/tree-ring-memory-cli/src/actions/remember.rs index e8f2609..9781652 100644 --- a/crates/tree-ring-memory-cli/src/actions/remember.rs +++ b/crates/tree-ring-memory-cli/src/actions/remember.rs @@ -1,5 +1,5 @@ -use tree_ring_memory_core::{MemoryEvent, SensitivityGuard}; -use tree_ring_memory_sqlite::SQLiteMemoryStore; +use tree_ring_memory_core::{MemoryEvent, MemorySource, SensitivityGuard}; +use tree_ring_memory_sqlite::{PutOutcome, SQLiteMemoryStore}; use super::ActionResult; @@ -10,12 +10,18 @@ pub struct RememberRequest { pub ring: String, pub scope: String, pub project: Option, + pub agent_profile: Option, + pub workflow_id: Option, + pub session_id: Option, + pub operation_id: Option, + pub source_ref: Option, pub tags: Vec, } #[derive(Debug, Clone, PartialEq)] pub struct RememberReport { pub memory: MemoryEvent, + pub created: bool, } pub fn remember( @@ -31,6 +37,11 @@ pub fn remember( ] .into_iter() .chain(request.project.iter().map(String::as_str)) + .chain(request.agent_profile.iter().map(String::as_str)) + .chain(request.workflow_id.iter().map(String::as_str)) + .chain(request.session_id.iter().map(String::as_str)) + .chain(request.operation_id.iter().map(String::as_str)) + .chain(request.source_ref.iter().map(String::as_str)) .chain(request.tags.iter().map(String::as_str)); let detected_sensitivity = guard .detect_text_sensitivity(values) @@ -40,13 +51,62 @@ pub fn remember( event.ring = request.ring; event.scope = request.scope; event.project = request.project; + event.agent_profile = request.agent_profile; + event.workflow_id = request.workflow_id; + event.session_id = request.session_id; + event.operation_id = request.operation_id; + if let Some(source_ref) = request.source_ref { + event.source = MemorySource { + source_type: "agent".to_string(), + ref_: source_ref, + quote: String::new(), + }; + } event.tags = request.tags; if detected_sensitivity != "normal" { event.sensitivity = detected_sensitivity; } event.validate().map_err(|err| err.to_string())?; - store.put(&event).map_err(|err| err.to_string())?; - Ok(RememberReport { memory: event }) + let (memory, created) = store_event_idempotently(store, &event)?; + Ok(RememberReport { memory, created }) +} + +pub fn store_event_idempotently( + store: &mut SQLiteMemoryStore, + event: &MemoryEvent, +) -> ActionResult<(MemoryEvent, bool)> { + match store.put_idempotent(event).map_err(|err| err.to_string())? { + PutOutcome::Created => Ok((event.clone(), true)), + PutOutcome::Existing(existing) if same_write_intent(&existing, event) => { + Ok((existing, false)) + } + PutOutcome::Existing(_) => Err(format!( + "operation_id {} is already bound to a different memory write", + event.operation_id.as_deref().unwrap_or("") + )), + } +} + +fn same_write_intent(existing: &MemoryEvent, requested: &MemoryEvent) -> bool { + existing.project == requested.project + && existing.agent_profile == requested.agent_profile + && existing.workflow_id == requested.workflow_id + && existing.session_id == requested.session_id + && existing.operation_id == requested.operation_id + && existing.scope == requested.scope + && existing.ring == requested.ring + && existing.event_type == requested.event_type + && existing.summary == requested.summary + && existing.details == requested.details + && existing.source == requested.source + && existing.tags == requested.tags + && existing.salience == requested.salience + && existing.confidence == requested.confidence + && existing.sensitivity == requested.sensitivity + && existing.retention == requested.retention + && existing.expires_at == requested.expires_at + && existing.supersedes == requested.supersedes + && existing.links == requested.links } #[cfg(test)] @@ -68,6 +128,11 @@ mod tests { ring: "cambium".to_string(), scope: "project".to_string(), project: Some("tree-ring".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: vec!["refactor".to_string()], }, ) @@ -94,6 +159,11 @@ mod tests { ring: "cambium".to_string(), scope: "global".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: Vec::new(), }, ) @@ -102,4 +172,82 @@ mod tests { let stored = store.get(&report.memory.id).unwrap().unwrap(); assert_eq!(stored.sensitivity, "health"); } + + #[test] + fn remember_action_round_trips_multi_agent_context_and_source() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + + let report = remember( + &mut store, + RememberRequest { + summary: "Worker found the failing boundary.".to_string(), + event_type: "lesson".to_string(), + ring: "cambium".to_string(), + scope: "agent".to_string(), + project: Some("tree-ring".to_string()), + agent_profile: Some("reviewer-2".to_string()), + workflow_id: Some("fanout-42".to_string()), + session_id: Some("attempt-1".to_string()), + operation_id: Some("finding-storage-lock".to_string()), + source_ref: Some("runs/fanout-42/reviewer-2.json".to_string()), + tags: vec!["storage".to_string()], + }, + ) + .unwrap(); + + assert!(report.created); + assert_eq!(report.memory.agent_profile.as_deref(), Some("reviewer-2")); + assert_eq!(report.memory.workflow_id.as_deref(), Some("fanout-42")); + assert_eq!(report.memory.session_id.as_deref(), Some("attempt-1")); + assert_eq!( + report.memory.operation_id.as_deref(), + Some("finding-storage-lock") + ); + assert_eq!(report.memory.source.ref_, "runs/fanout-42/reviewer-2.json"); + } + + #[test] + fn operation_id_replay_is_idempotent_and_conflicts_fail_closed() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let request = RememberRequest { + summary: "One logical worker result.".to_string(), + event_type: "lesson".to_string(), + ring: "cambium".to_string(), + scope: "workflow".to_string(), + project: Some("tree-ring".to_string()), + agent_profile: Some("worker-1".to_string()), + workflow_id: Some("fanout-42".to_string()), + session_id: None, + operation_id: Some("task-7".to_string()), + source_ref: None, + tags: Vec::new(), + }; + + let first = remember(&mut store, request.clone()).unwrap(); + let replay = remember(&mut store, request).unwrap(); + let mut conflict_request = RememberRequest { + summary: "Conflicting worker result.".to_string(), + event_type: "lesson".to_string(), + ring: "cambium".to_string(), + scope: "workflow".to_string(), + project: Some("tree-ring".to_string()), + agent_profile: Some("worker-1".to_string()), + workflow_id: Some("fanout-42".to_string()), + session_id: None, + operation_id: Some("task-7".to_string()), + source_ref: None, + tags: Vec::new(), + }; + let conflict = remember(&mut store, conflict_request.clone()).unwrap_err(); + conflict_request.operation_id = Some("task-8".to_string()); + + assert!(first.created); + assert!(!replay.created); + assert_eq!(first.memory.id, replay.memory.id); + assert!(conflict.contains("already bound")); + assert_eq!(store.list_all(true).unwrap().len(), 1); + assert!(remember(&mut store, conflict_request).unwrap().created); + } } diff --git a/crates/tree-ring-memory-cli/src/agent_awareness.rs b/crates/tree-ring-memory-cli/src/agent_awareness.rs index 735696d..7a05685 100644 --- a/crates/tree-ring-memory-cli/src/agent_awareness.rs +++ b/crates/tree-ring-memory-cli/src/agent_awareness.rs @@ -61,6 +61,13 @@ Adapter rules: - Run adapter commands with `--dry-run` before writing memory. - `tree-ring integrations scan` is read-only; add harness bridge references manually until a link command is available. +Multi-agent coordination: + +- Give each worker a distinct `--agent-profile`, share one `--workflow-id`, and use a new `--session-id` for each execution attempt. +- Give every logical write a stable `--operation-id` and a durable `--source-ref`; exact retries return the original memory, while conflicting reuse fails closed. +- At fan-in, recall with the shared workflow/session and an explicit scope. Scope and identity fields partition and route local memory; they are not access-control boundaries. +- A shared SQLite root supports concurrent processes on one host and a local filesystem. Use per-host stores plus an explicit source-preserving fan-in for cross-host or network-filesystem workflows. + Memory quality gates: - Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds. @@ -290,6 +297,19 @@ call `tree-ring recall`, `tree-ring remember`, `tree-ring evidence`, `tree-ring forget`, `tree-ring consolidate --dry-run`, or `tree-ring maintain`. They do not authorize hidden transcript scraping or autonomous durable writes. +## Multi-Agent Coordination + +For same-host fan-out/fan-in, give every worker a distinct `--agent-profile`, +share one `--workflow-id`, use a new `--session-id` for each attempt, and attach +a stable `--operation-id` plus `--source-ref` to every logical write. At fan-in, +recall with the workflow, session, and intended scope before creating a +source-linked shared summary. + +Scope and identity fields partition and route local memory; they are not +authorization boundaries. A shared SQLite root is for concurrent processes on +one host using a local filesystem. Cross-host or network-filesystem workflows +should use per-host stores and an explicit evidence-preserving fan-in. + ## Memory Quality Gates Recall gates: @@ -418,6 +438,9 @@ mod tests { assert!(agents.contains("revolve sync --source-root")); assert!(agents.contains("integrations scan --source-root")); assert!(agents.contains("Tree Ring Memory Project Contract")); + assert!(agents.contains("Multi-Agent Coordination")); + assert!(agents.contains("--operation-id")); + assert!(agents.contains("not authorization boundaries")); } #[test] @@ -444,6 +467,9 @@ mod tests { let cli = fs::read_to_string(root.join("CLI.md")).unwrap(); assert!(cli.contains("Memory quality gates")); + assert!(cli.contains("Multi-agent coordination")); + assert!(cli.contains("--workflow-id")); + assert!(cli.contains("not access-control boundaries")); assert!(cli.contains("Before substantial project work, recall project constraints, scars, user preferences, and unresolved seeds.")); assert!(cli .contains("Before risky changes, recall warnings and evidence-linked prior failures.")); diff --git a/crates/tree-ring-memory-cli/src/evidence.rs b/crates/tree-ring-memory-cli/src/evidence.rs index 0f68cd7..23f5ea2 100644 --- a/crates/tree-ring-memory-cli/src/evidence.rs +++ b/crates/tree-ring-memory-cli/src/evidence.rs @@ -1,8 +1,12 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::BTreeMap; -use std::fs; +use std::fs::{self, OpenOptions}; +use std::io::Write; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static ATOMIC_WRITE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -202,10 +206,7 @@ fn load_index(index_path: &Path) -> Result { serde_json::from_str(&input).map_err(|err| err.to_string()) } -pub(crate) fn read_or_create_index( - evidence_dir: &Path, - generated_at: &str, -) -> Result { +fn read_or_create_index(evidence_dir: &Path, generated_at: &str) -> Result { let index_path = evidence_dir.join("evidence-index.json"); if index_path.exists() { let input = fs::read_to_string(&index_path).map_err(|err| err.to_string())?; @@ -222,14 +223,83 @@ pub(crate) fn read_or_create_index( }) } -pub(crate) fn write_index(evidence_dir: &Path, index: &EvidenceIndex) -> Result { +fn write_index(evidence_dir: &Path, index: &EvidenceIndex) -> Result { fs::create_dir_all(evidence_dir).map_err(|err| err.to_string())?; let index_path = evidence_dir.join("evidence-index.json"); let json = serde_json::to_string_pretty(index).map_err(|err| err.to_string())?; - fs::write(&index_path, json).map_err(|err| err.to_string())?; + atomic_write(&index_path, json.as_bytes())?; Ok(index_path) } +/// Publishes evidence payloads and their index update while holding the same +/// per-directory lock. +/// +/// Callers must perform every write to a fixed artifact path from `publish`. +/// This prevents concurrent producers from leaving an index record from one +/// run pointing at a payload overwritten by another run. +pub(crate) fn publish_indexed_evidence( + evidence_dir: &Path, + generated_at: &str, + publish: impl FnOnce(&mut EvidenceIndex) -> Result<(), String>, +) -> Result { + fs::create_dir_all(evidence_dir).map_err(|err| err.to_string())?; + let lock_path = evidence_dir.join(".evidence-index.lock"); + let lock_file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&lock_path) + .map_err(|err| err.to_string())?; + lock_file.lock().map_err(|err| err.to_string())?; + + let mut index = read_or_create_index(evidence_dir, generated_at)?; + if let Some(certification) = certification_record_from_metrics(evidence_dir, generated_at) { + index.certification = Some(certification); + } + publish(&mut index)?; + write_index(evidence_dir, &index) +} + +pub(crate) fn atomic_write(path: &Path, contents: &[u8]) -> Result<(), String> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).map_err(|err| err.to_string())?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + format!( + "atomic output path has no UTF-8 file name: {}", + path.display() + ) + })?; + let counter = ATOMIC_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); + let temp_path = parent.join(format!( + ".{file_name}.{}.{}.tmp", + std::process::id(), + counter + )); + + let result = (|| { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temp_path) + .map_err(|err| err.to_string())?; + file.write_all(contents).map_err(|err| err.to_string())?; + file.sync_all().map_err(|err| err.to_string())?; + drop(file); + fs::rename(&temp_path, path).map_err(|err| err.to_string()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temp_path); + } + result +} + pub(crate) fn rollup_index_status(index: &EvidenceIndex) -> EvidenceStatus { if index .harness @@ -379,6 +449,16 @@ fn certification_record_from_metrics( if !metrics_path.exists() { return None; } + let metrics_generated_at = fs::read_to_string(&metrics_path) + .ok() + .and_then(|input| serde_json::from_str::(&input).ok()) + .and_then(|value| { + value + .get("created_at") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) + .unwrap_or_else(|| generated_at.to_string()); let summary_path = evidence_dir.join("summary.md"); Some(EvidenceRecordRef { category: "certification".to_string(), @@ -386,7 +466,7 @@ fn certification_record_from_metrics( label: "Local certification".to_string(), path: PathBuf::from("metrics.json"), summary_path: summary_path.exists().then_some(PathBuf::from("summary.md")), - generated_at: generated_at.to_string(), + generated_at: metrics_generated_at, }) } @@ -492,6 +572,9 @@ fn get_value<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> { #[cfg(test)] mod tests { use super::*; + use std::sync::{mpsc, Arc, Barrier}; + use std::thread; + use std::time::Duration; use tempfile::tempdir; #[test] @@ -825,4 +908,156 @@ mod tests { index.recall_quality.as_mut().unwrap().status = EvidenceStatus::Fail; assert_eq!(rollup_index_status(&index), EvidenceStatus::Fail); } + + #[test] + fn concurrent_index_updates_preserve_every_producer() { + const PRODUCERS: usize = 24; + + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("evidence"); + let barrier = Arc::new(Barrier::new(PRODUCERS)); + let handles = (0..PRODUCERS) + .map(|producer| { + let evidence_dir = evidence_dir.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + barrier.wait(); + let harness_id = format!("worker-{producer:02}"); + publish_indexed_evidence(&evidence_dir, "2026-07-23T00:00:00Z", |index| { + index.harness.insert( + harness_id.clone(), + EvidenceRecordRef { + category: "harness".to_string(), + status: EvidenceStatus::Pass, + label: format!("Worker {producer:02}"), + path: PathBuf::from(format!("harness/{harness_id}.json")), + summary_path: None, + generated_at: "2026-07-23T00:00:00Z".to_string(), + }, + ); + Ok(()) + }) + .unwrap(); + }) + }) + .collect::>(); + + for handle in handles { + handle.join().unwrap(); + } + + let index = load_index(&evidence_dir.join("evidence-index.json")).unwrap(); + assert_eq!(index.harness.len(), PRODUCERS); + for producer in 0..PRODUCERS { + assert!(index.harness.contains_key(&format!("worker-{producer:02}"))); + } + } + + #[test] + fn concurrent_same_target_publications_keep_payload_and_index_from_the_same_run() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("evidence"); + let payload_path = evidence_dir.join("harness/shared.json"); + let (first_entered_tx, first_entered_rx) = mpsc::channel(); + let (release_first_tx, release_first_rx) = mpsc::channel(); + + let first_evidence_dir = evidence_dir.clone(); + let first_payload_path = payload_path.clone(); + let first = thread::spawn(move || { + publish_indexed_evidence(&first_evidence_dir, "2026-07-23T00:00:01Z", |index| { + atomic_write(&first_payload_path, br#"{"producer":"first"}"#)?; + first_entered_tx.send(()).unwrap(); + release_first_rx.recv().unwrap(); + index.harness.insert( + "shared".to_string(), + EvidenceRecordRef { + category: "harness".to_string(), + status: EvidenceStatus::Pass, + label: "first".to_string(), + path: PathBuf::from("harness/shared.json"), + summary_path: None, + generated_at: "2026-07-23T00:00:01Z".to_string(), + }, + ); + Ok(()) + }) + .unwrap(); + }); + + first_entered_rx.recv().unwrap(); + + let second_evidence_dir = evidence_dir.clone(); + let second_payload_path = payload_path.clone(); + let (second_attempting_tx, second_attempting_rx) = mpsc::channel(); + let (second_done_tx, second_done_rx) = mpsc::channel(); + let second = thread::spawn(move || { + second_attempting_tx.send(()).unwrap(); + publish_indexed_evidence(&second_evidence_dir, "2026-07-23T00:00:02Z", |index| { + atomic_write(&second_payload_path, br#"{"producer":"second"}"#)?; + index.harness.insert( + "shared".to_string(), + EvidenceRecordRef { + category: "harness".to_string(), + status: EvidenceStatus::Fail, + label: "second".to_string(), + path: PathBuf::from("harness/shared.json"), + summary_path: None, + generated_at: "2026-07-23T00:00:02Z".to_string(), + }, + ); + Ok(()) + }) + .unwrap(); + second_done_tx.send(()).unwrap(); + }); + + second_attempting_rx.recv().unwrap(); + assert!( + second_done_rx + .recv_timeout(Duration::from_millis(50)) + .is_err(), + "the second publication must wait for the first publication lock" + ); + release_first_tx.send(()).unwrap(); + first.join().unwrap(); + second.join().unwrap(); + second_done_rx.recv().unwrap(); + + let payload: Value = + serde_json::from_str(&fs::read_to_string(&payload_path).unwrap()).unwrap(); + let index = load_index(&evidence_dir.join("evidence-index.json")).unwrap(); + let record = index.harness.get("shared").unwrap(); + assert_eq!(payload["producer"], "second"); + assert_eq!(record.label, "second"); + assert_eq!(record.status, EvidenceStatus::Fail); + assert_eq!(record.generated_at, "2026-07-23T00:00:02Z"); + } + + #[test] + fn locked_index_update_publishes_current_certification_metrics() { + let dir = tempdir().unwrap(); + let evidence_dir = dir.path().join("evidence"); + fs::create_dir_all(&evidence_dir).unwrap(); + fs::write( + evidence_dir.join("metrics.json"), + r#"{"ok":true,"created_at":"2026-07-23T00:00:00Z"}"#, + ) + .unwrap(); + fs::write(evidence_dir.join("summary.md"), "# Current run\n").unwrap(); + + publish_indexed_evidence(&evidence_dir, "2026-07-23T00:01:00Z", |index| { + index.generated_at = "2026-07-23T00:01:00Z".to_string(); + Ok(()) + }) + .unwrap(); + + let index = load_index(&evidence_dir.join("evidence-index.json")).unwrap(); + let certification = index.certification.unwrap(); + assert_eq!(certification.path, PathBuf::from("metrics.json")); + assert_eq!( + certification.summary_path, + Some(PathBuf::from("summary.md")) + ); + assert_eq!(certification.generated_at, "2026-07-23T00:00:00Z"); + } } diff --git a/crates/tree-ring-memory-cli/src/harness_evidence.rs b/crates/tree-ring-memory-cli/src/harness_evidence.rs index d75a699..d5e4fe6 100644 --- a/crates/tree-ring-memory-cli/src/harness_evidence.rs +++ b/crates/tree-ring-memory-cli/src/harness_evidence.rs @@ -1,5 +1,5 @@ use crate::evidence::{ - read_or_create_index, rollup_index_status, write_index, EvidenceRecordRef, EvidenceStatus, + atomic_write, publish_indexed_evidence, rollup_index_status, EvidenceRecordRef, EvidenceStatus, }; use crate::integrations::{scan_integrations, IntegrationMarker, MarkerOrigin}; use chrono::{SecondsFormat, Utc}; @@ -70,9 +70,6 @@ pub fn certify_harnesses( let generated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); let report = scan_integrations(&request.source_root); let guidance = inspect_guidance(&request.source_root); - let harness_dir = request.evidence_dir.join("harness"); - fs::create_dir_all(&harness_dir).map_err(|err| err.to_string())?; - let mut records = Vec::new(); for harness_id in CERTIFIED_HARNESS_IDS { let integration = report @@ -81,13 +78,10 @@ pub fn certify_harnesses( .find(|integration| integration.id == *harness_id) .ok_or_else(|| format!("missing integration definition for {harness_id}"))?; let record = probe_record(integration, &request.source_root, &generated_at, &guidance); - let path = harness_dir.join(format!("{harness_id}.json")); - let json = serde_json::to_string_pretty(&record).map_err(|err| err.to_string())?; - fs::write(&path, json).map_err(|err| err.to_string())?; records.push(record); } - let index_path = merge_harness_index(&request.evidence_dir, &generated_at, &records)?; + let index_path = publish_harness_evidence(&request.evidence_dir, &generated_at, &records)?; let pass_count = records .iter() .filter(|record| record.status == EvidenceStatus::Pass) @@ -212,35 +206,45 @@ fn marker_from_scan(marker: &IntegrationMarker) -> HarnessProbeMarker { } } -fn merge_harness_index( +fn publish_harness_evidence( evidence_dir: &Path, generated_at: &str, records: &[HarnessProbeRecord], ) -> Result { - let mut index = read_or_create_index(evidence_dir, generated_at)?; - index.generated_at = generated_at.to_string(); - for record in records { - index.harness.insert( - record.harness_id.clone(), - EvidenceRecordRef { - category: "harness".to_string(), - status: record.status, - label: record.name.clone(), - path: PathBuf::from(format!("harness/{}.json", record.harness_id)), - summary_path: None, - generated_at: record.generated_at.clone(), - }, - ); - } - index.missing.retain(|item| item != "harness"); - if index.recall_quality.is_none() && !index.missing.iter().any(|item| item == "recall_quality") - { - index.missing.push("recall_quality".to_string()); - } - index.missing.sort(); - index.missing.dedup(); - index.overall_status = rollup_index_status(&index); - write_index(evidence_dir, &index) + publish_indexed_evidence(evidence_dir, generated_at, |index| { + let harness_dir = evidence_dir.join("harness"); + fs::create_dir_all(&harness_dir).map_err(|err| err.to_string())?; + for record in records { + let path = harness_dir.join(format!("{}.json", record.harness_id)); + let json = serde_json::to_string_pretty(record).map_err(|err| err.to_string())?; + atomic_write(&path, json.as_bytes())?; + } + + index.generated_at = generated_at.to_string(); + for record in records { + index.harness.insert( + record.harness_id.clone(), + EvidenceRecordRef { + category: "harness".to_string(), + status: record.status, + label: record.name.clone(), + path: PathBuf::from(format!("harness/{}.json", record.harness_id)), + summary_path: None, + generated_at: record.generated_at.clone(), + }, + ); + } + index.missing.retain(|item| item != "harness"); + if index.recall_quality.is_none() + && !index.missing.iter().any(|item| item == "recall_quality") + { + index.missing.push("recall_quality".to_string()); + } + index.missing.sort(); + index.missing.dedup(); + index.overall_status = rollup_index_status(index); + Ok(()) + }) } #[cfg(test)] diff --git a/crates/tree-ring-memory-cli/src/main.rs b/crates/tree-ring-memory-cli/src/main.rs index 68c5d92..46a4ad3 100644 --- a/crates/tree-ring-memory-cli/src/main.rs +++ b/crates/tree-ring-memory-cli/src/main.rs @@ -20,7 +20,7 @@ use actions::lifecycle::{ MaintainActionRequest, }; use actions::recall::{recall as recall_action, RecallRequest}; -use actions::remember::{remember as remember_action, RememberRequest}; +use actions::remember::{remember as remember_action, store_event_idempotently, RememberRequest}; use harness_evidence::{ certify_harnesses, HarnessCertificationReport, HarnessCertificationRequest, }; @@ -77,6 +77,34 @@ enum Command { scope: String, #[arg(long)] project: Option, + #[arg( + long, + env = "TREE_RING_AGENT_PROFILE", + help = "agent role or worker identity that produced this memory" + )] + agent_profile: Option, + #[arg( + long, + env = "TREE_RING_WORKFLOW_ID", + help = "shared workflow or fan-out/fan-in correlation id" + )] + workflow_id: Option, + #[arg( + long, + env = "TREE_RING_SESSION_ID", + help = "session or execution-attempt correlation id" + )] + session_id: Option, + #[arg( + long, + help = "idempotency key for one logical write within its project/workflow/agent namespace" + )] + operation_id: Option, + #[arg( + long, + help = "source artifact, task, run, message, or result reference" + )] + source_ref: Option, #[arg(long = "tag")] tags: Vec, }, @@ -96,6 +124,26 @@ enum Command { evidence_ref: String, #[arg(long, help = "optional project scope")] project: Option, + #[arg( + long, + env = "TREE_RING_AGENT_PROFILE", + help = "agent role or worker identity that produced this evidence" + )] + agent_profile: Option, + #[arg( + long, + env = "TREE_RING_WORKFLOW_ID", + help = "shared workflow or fan-out/fan-in correlation id" + )] + workflow_id: Option, + #[arg( + long, + env = "TREE_RING_SESSION_ID", + help = "session or execution-attempt correlation id" + )] + session_id: Option, + #[arg(long, help = "idempotency key for one logical evidence write")] + operation_id: Option, #[arg(long, help = "optional extra context")] details: Option, #[arg(long, help = "optional numeric evaluation score")] @@ -108,6 +156,29 @@ enum Command { query: String, #[arg(long)] project: Option, + #[arg( + long, + env = "TREE_RING_AGENT_PROFILE", + help = "return only memories attributed to this agent profile" + )] + agent_profile: Option, + #[arg( + long, + env = "TREE_RING_WORKFLOW_ID", + help = "return only memories from this workflow" + )] + workflow_id: Option, + #[arg( + long, + env = "TREE_RING_SESSION_ID", + help = "return only memories from this session" + )] + session_id: Option, + #[arg( + long, + help = "return only this scope (global, project, agent, session, workflow, tool, eval, manual, dox, revolve)" + )] + scope: Option, #[arg(long, default_value_t = 8)] limit: usize, #[arg(long)] @@ -162,6 +233,16 @@ enum Command { period_key: Option, #[arg(long, help = "optional project filter")] project: Option, + #[arg( + long, + env = "TREE_RING_AGENT_PROFILE", + help = "optional agent-profile filter" + )] + agent_profile: Option, + #[arg(long, env = "TREE_RING_WORKFLOW_ID", help = "optional workflow filter")] + workflow_id: Option, + #[arg(long, env = "TREE_RING_SESSION_ID", help = "optional session filter")] + session_id: Option, #[arg(long, help = "plan consolidation without writing summaries or records")] dry_run: bool, #[arg( @@ -434,6 +515,9 @@ fn run(cli: Cli) -> Result<(), String> { period_type, period_key, project, + agent_profile, + workflow_id, + session_id, force, dry_run: true, } = &cli.command @@ -444,6 +528,9 @@ fn run(cli: Cli) -> Result<(), String> { period_type: period_type.clone(), period_key: period_key.clone(), project: project.clone(), + agent_profile: agent_profile.clone(), + workflow_id: workflow_id.clone(), + session_id: session_id.clone(), dry_run: true, force: *force, }, @@ -545,6 +632,11 @@ fn run(cli: Cli) -> Result<(), String> { ring, scope, project, + agent_profile, + workflow_id, + session_id, + operation_id, + source_ref, tags, } => { let report = remember_action( @@ -555,6 +647,11 @@ fn run(cli: Cli) -> Result<(), String> { ring, scope, project, + agent_profile, + workflow_id, + session_id, + operation_id, + source_ref, tags, }, )?; @@ -572,20 +669,28 @@ fn run(cli: Cli) -> Result<(), String> { outcome, evidence_ref, project, + agent_profile, + workflow_id, + session_id, + operation_id, details, score, tags, } => { - let event = evidence_event( + let event = evidence_event(EvidenceEventRequest { summary, outcome, evidence_ref, project, + agent_profile, + workflow_id, + session_id, + operation_id, details, score, tags, - )?; - store.put(&event).map_err(|err| err.to_string())?; + })?; + let (event, _created) = store_event_idempotently(&mut store, &event)?; if cli.json { println!( "{}", @@ -601,6 +706,10 @@ fn run(cli: Cli) -> Result<(), String> { Command::Recall { query, project, + agent_profile, + workflow_id, + session_id, + scope, limit, include_sensitive, } => { @@ -609,6 +718,10 @@ fn run(cli: Cli) -> Result<(), String> { RecallRequest { query, project, + agent_profile, + workflow_id, + session_id, + scope, limit, include_sensitive, include_superseded: false, @@ -670,6 +783,9 @@ fn run(cli: Cli) -> Result<(), String> { period_type, period_key, project, + agent_profile, + workflow_id, + session_id, dry_run, force, } => { @@ -679,6 +795,9 @@ fn run(cli: Cli) -> Result<(), String> { period_type, period_key, project, + agent_profile, + workflow_id, + session_id, dry_run, force, }, @@ -755,15 +874,35 @@ fn run(cli: Cli) -> Result<(), String> { Ok(()) } -fn evidence_event( +struct EvidenceEventRequest { summary: String, outcome: String, evidence_ref: String, project: Option, + agent_profile: Option, + workflow_id: Option, + session_id: Option, + operation_id: Option, details: Option, score: Option, tags: Vec, -) -> Result { +} + +fn evidence_event(request: EvidenceEventRequest) -> Result { + let EvidenceEventRequest { + summary, + outcome, + evidence_ref, + project, + agent_profile, + workflow_id, + session_id, + operation_id, + details, + score, + tags, + } = request; + if summary.trim().is_empty() { return Err("evidence summary is required".to_string()); } @@ -817,6 +956,10 @@ fn evidence_event( let values = [&summary, &outcome, &evidence_ref] .into_iter() .chain(project.iter()) + .chain(agent_profile.iter()) + .chain(workflow_id.iter()) + .chain(session_id.iter()) + .chain(operation_id.iter()) .chain(details.iter()) .chain(tags.iter()) .map(String::as_str); @@ -828,6 +971,10 @@ fn evidence_event( event.ring = ring.to_string(); event.scope = "eval".to_string(); event.project = project; + event.agent_profile = agent_profile; + event.workflow_id = workflow_id; + event.session_id = session_id; + event.operation_id = operation_id; event.details = evidence_details(&normalized_outcome, score, details); event.source.source_type = "evidence".to_string(); event.source.ref_ = evidence_ref.trim().to_string(); @@ -1226,6 +1373,11 @@ mod tests { ring: "cambium".to_string(), scope: "global".to_string(), project: Some("sk-proj-abcdefghijklmnopqrstuvwxyz1234567890".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: Vec::new(), }, }) @@ -1254,6 +1406,11 @@ mod tests { ring: "cambium".to_string(), scope: "global".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: Vec::new(), }, }) @@ -1699,6 +1856,10 @@ mod tests { outcome: "promoted".to_string(), evidence_ref: "evals/chat-state/run-042".to_string(), project: Some("agent-ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, details: Some("Passed regression suite and manual replay.".to_string()), score: Some(0.91), tags: vec!["chat".to_string()], @@ -1733,6 +1894,10 @@ mod tests { outcome: "rejected".to_string(), evidence_ref: "evals/cache-branch/run-013".to_string(), project: Some("agent-ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, details: None, score: Some(0.82), tags: Vec::new(), @@ -1758,6 +1923,10 @@ mod tests { outcome: "observed".to_string(), evidence_ref: "evals/run".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, details: None, score: Some(2.0), tags: Vec::new(), @@ -1896,6 +2065,11 @@ mod tests { ring: "cambium".to_string(), scope: "global".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: vec!["private diagnosis".to_string()], }, }) @@ -1948,6 +2122,11 @@ mod tests { ring: "cambium".to_string(), scope: "global".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: Vec::new(), }, }) @@ -1991,7 +2170,7 @@ mod tests { } #[test] - fn audit_existing_store_does_not_create_sqlite_sidecars() { + fn audit_existing_store_does_not_mutate_database_contents() { let dir = tempdir().unwrap(); let root = dir.path().join(".tree-ring"); let db_path = root.join("memory.sqlite"); @@ -2014,6 +2193,7 @@ mod tests { assert!(db_path.exists()); assert!(!wal_path.exists()); assert!(!shm_path.exists()); + let before = fs::read(&db_path).unwrap(); run(Cli { root: root.clone(), @@ -2024,8 +2204,7 @@ mod tests { }) .unwrap(); - assert!(!wal_path.exists()); - assert!(!shm_path.exists()); + assert_eq!(fs::read(&db_path).unwrap(), before); } #[test] @@ -2040,6 +2219,9 @@ mod tests { period_type: "manual".to_string(), period_key: Some("manual-test".to_string()), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: true, force: false, }, @@ -2061,6 +2243,9 @@ mod tests { period_type: "manual".to_string(), period_key: Some("manual-empty".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }, @@ -2093,6 +2278,11 @@ mod tests { ring: "cambium".to_string(), scope: "global".to_string(), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: vec!["memory".to_string()], }, }) @@ -2105,6 +2295,9 @@ mod tests { period_type: "manual".to_string(), period_key: Some("manual-test".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }, @@ -2116,6 +2309,9 @@ mod tests { period_type: ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }) diff --git a/crates/tree-ring-memory-cli/src/recall_quality.rs b/crates/tree-ring-memory-cli/src/recall_quality.rs index 65588af..ebc4b85 100644 --- a/crates/tree-ring-memory-cli/src/recall_quality.rs +++ b/crates/tree-ring-memory-cli/src/recall_quality.rs @@ -1,5 +1,5 @@ use crate::evidence::{ - read_or_create_index, rollup_index_status, write_index, EvidenceRecordRef, EvidenceStatus, + atomic_write, publish_indexed_evidence, rollup_index_status, EvidenceRecordRef, EvidenceStatus, }; use chrono::{SecondsFormat, Utc}; use serde::{Deserialize, Serialize}; @@ -269,30 +269,31 @@ fn write_report_and_index( let report_dir = evidence_dir.join("recall-quality"); fs::create_dir_all(&report_dir).map_err(|err| err.to_string())?; let report_json = serde_json::to_string_pretty(report).map_err(|err| err.to_string())?; - fs::write(&report.record_path, report_json).map_err(|err| err.to_string())?; - - let mut index = read_or_create_index(evidence_dir, generated_at)?; - index.generated_at = generated_at.to_string(); - index.recall_quality = Some(EvidenceRecordRef { - category: "recall_quality".to_string(), - status: report.status, - label: "Recall quality".to_string(), - path: PathBuf::from(format!("recall-quality/{RECALL_QUALITY_QUERY_SET_ID}.json")), - summary_path: None, - generated_at: generated_at.to_string(), - }); - index.missing.retain(|item| item != "recall_quality"); - if index.harness.is_empty() { - if !index.missing.iter().any(|item| item == "harness") { - index.missing.push("harness".to_string()); + + publish_indexed_evidence(evidence_dir, generated_at, |index| { + atomic_write(&report.record_path, report_json.as_bytes())?; + index.generated_at = generated_at.to_string(); + index.recall_quality = Some(EvidenceRecordRef { + category: "recall_quality".to_string(), + status: report.status, + label: "Recall quality".to_string(), + path: PathBuf::from(format!("recall-quality/{RECALL_QUALITY_QUERY_SET_ID}.json")), + summary_path: None, + generated_at: generated_at.to_string(), + }); + index.missing.retain(|item| item != "recall_quality"); + if index.harness.is_empty() { + if !index.missing.iter().any(|item| item == "harness") { + index.missing.push("harness".to_string()); + } + } else { + index.missing.retain(|item| item != "harness"); } - } else { - index.missing.retain(|item| item != "harness"); - } - index.missing.sort(); - index.missing.dedup(); - index.overall_status = rollup_index_status(&index); - write_index(evidence_dir, &index)?; + index.missing.sort(); + index.missing.dedup(); + index.overall_status = rollup_index_status(index); + Ok(()) + })?; Ok(()) } diff --git a/crates/tree-ring-memory-cli/src/tui/app.rs b/crates/tree-ring-memory-cli/src/tui/app.rs index ba82908..d8eabb1 100644 --- a/crates/tree-ring-memory-cli/src/tui/app.rs +++ b/crates/tree-ring-memory-cli/src/tui/app.rs @@ -1,7 +1,7 @@ use std::path::{Component, Path, PathBuf}; use ratatui::crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; -use tree_ring_memory_core::{now_iso, ConsolidationRequest, MemoryEvent}; +use tree_ring_memory_core::{ConsolidationRequest, MemoryEvent}; use tree_ring_memory_sqlite::{MemoryRetriever, RecallResult, SQLiteMemoryStore}; use crate::actions::export_import::{export_jsonl, ExportActionRequest}; @@ -294,6 +294,11 @@ impl App { ring: "cambium".to_string(), scope: "global".to_string(), project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, + source_ref: None, tags: Vec::new(), }, )?; @@ -343,17 +348,13 @@ impl App { } ActionKind::ChangeRing { ring, event_type } => { if let Some(memory_id) = pending.memory_id { - if let Some(mut memory) = - self.store.get(&memory_id).map_err(|err| err.to_string())? + match self + .store + .change_ring(&memory_id, &ring, &event_type) + .map_err(|err| err.to_string())? { - memory.ring = ring.clone(); - memory.event_type = event_type; - memory.updated_at = now_iso(); - if ring == "heartwood" { - memory.retention = "durable".to_string(); - } - self.store.put(&memory).map_err(|err| err.to_string())?; - self.status = format!("marked {memory_id} as {ring}"); + Some(_) => self.status = format!("marked {memory_id} as {ring}"), + None => self.status = format!("memory not found: {memory_id}"), } } } @@ -651,6 +652,23 @@ mod tests { assert_eq!(app.dashboard.total, 0); } + #[test] + fn confirmation_reports_when_change_ring_target_is_missing() { + let dir = tempdir().unwrap(); + let mut app = app(&dir); + app.status = "marked stale memory as heartwood".to_string(); + app.pending_action = Some(PendingAction::change_ring( + "mem_missing".to_string(), + "Missing".to_string(), + "heartwood", + "lesson", + )); + + confirm(&mut app); + + assert_eq!(app.status, "memory not found: mem_missing"); + } + #[test] fn slash_rings_switches_exploded_mode() { let dir = tempdir().unwrap(); diff --git a/crates/tree-ring-memory-cli/src/tui/stream.rs b/crates/tree-ring-memory-cli/src/tui/stream.rs index 67b5991..0579fc1 100644 --- a/crates/tree-ring-memory-cli/src/tui/stream.rs +++ b/crates/tree-ring-memory-cli/src/tui/stream.rs @@ -4,6 +4,9 @@ use std::path::PathBuf; use serde::Deserialize; +const MAX_EVENT_LINE_BYTES: usize = 256 * 1024; +const MAX_READ_BATCH_BYTES: u64 = 1024 * 1024; + #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct LiveEvent { #[serde(default)] @@ -17,6 +20,10 @@ pub struct LiveEvent { #[serde(default)] pub agent_profile: Option, #[serde(default)] + pub workflow_id: Option, + #[serde(default)] + pub session_id: Option, + #[serde(default)] pub count_delta: Option, #[serde(default)] pub label: Option, @@ -38,11 +45,18 @@ impl LiveEvent { pub struct EventStreamReader { path: PathBuf, offset: u64, + pending: Vec, + discarding_oversized_line: bool, } impl EventStreamReader { pub fn new(path: PathBuf) -> Self { - Self { path, offset: 0 } + Self { + path, + offset: 0, + pending: Vec::new(), + discarding_oversized_line: false, + } } pub fn read_new_events(&mut self) -> Result, String> { @@ -52,20 +66,48 @@ impl EventStreamReader { let file_len = file.metadata().map_err(|err| err.to_string())?.len(); if file_len < self.offset { self.offset = 0; + self.pending.clear(); + self.discarding_oversized_line = false; } file.seek(SeekFrom::Start(self.offset)) .map_err(|err| err.to_string())?; - let mut appended = String::new(); - file.read_to_string(&mut appended) + let mut appended = Vec::new(); + file.take(MAX_READ_BATCH_BYTES) + .read_to_end(&mut appended) .map_err(|err| err.to_string())?; - self.offset = file.stream_position().map_err(|err| err.to_string())?; + self.offset += appended.len() as u64; + + let appended = if self.discarding_oversized_line { + let Some(newline) = appended.iter().position(|byte| *byte == b'\n') else { + return Ok(Vec::new()); + }; + self.discarding_oversized_line = false; + &appended[newline + 1..] + } else { + &appended + }; + self.pending.extend_from_slice(appended); + let Some(last_newline) = self.pending.iter().rposition(|byte| *byte == b'\n') else { + if self.pending.len() > MAX_EVENT_LINE_BYTES { + self.pending.clear(); + self.discarding_oversized_line = true; + } + return Ok(Vec::new()); + }; + let remainder = self.pending.split_off(last_newline + 1); + let complete = std::mem::replace(&mut self.pending, remainder); + if self.pending.len() > MAX_EVENT_LINE_BYTES { + self.pending.clear(); + self.discarding_oversized_line = true; + } let mut events = Vec::new(); - for line in appended.lines() { - if line.trim().is_empty() { + for line in complete.split(|byte| *byte == b'\n') { + let line = line.strip_suffix(b"\r").unwrap_or(line); + if line.len() > MAX_EVENT_LINE_BYTES || line.iter().all(u8::is_ascii_whitespace) { continue; } - if let Ok(event) = serde_json::from_str::(line) { + if let Ok(event) = serde_json::from_slice::(line) { events.push(event); } } @@ -114,4 +156,52 @@ mod tests { assert_eq!(events.len(), 1); assert_eq!(events[0].safe_label(), "secret blocked"); } + + #[test] + fn retains_partial_final_line_until_a_writer_finishes_it() { + let mut file = NamedTempFile::new().unwrap(); + write!( + file, + r#"{{"event":"remembered","ring":"cambium","label":"worker "# + ) + .unwrap(); + file.flush().unwrap(); + let mut reader = EventStreamReader::new(file.path().to_path_buf()); + + assert!(reader.read_new_events().unwrap().is_empty()); + + writeln!(file, r#"résumé"}}"#).unwrap(); + file.flush().unwrap(); + let events = reader.read_new_events().unwrap(); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].safe_label(), "worker résumé"); + assert!(reader.read_new_events().unwrap().is_empty()); + } + + #[test] + fn bounds_partial_line_memory_and_recovers_at_the_next_event() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(&vec![b'x'; MAX_EVENT_LINE_BYTES + 1]) + .unwrap(); + file.flush().unwrap(); + let mut reader = EventStreamReader::new(file.path().to_path_buf()); + + assert!(reader.read_new_events().unwrap().is_empty()); + assert!(reader.pending.is_empty()); + assert!(reader.discarding_oversized_line); + + writeln!(file).unwrap(); + writeln!( + file, + r#"{{"event":"remembered","ring":"cambium","label":"recovered"}}"# + ) + .unwrap(); + file.flush().unwrap(); + let events = reader.read_new_events().unwrap(); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].safe_label(), "recovered"); + assert!(!reader.discarding_oversized_line); + } } diff --git a/crates/tree-ring-memory-cli/tests/multi_agent_acceptance.rs b/crates/tree-ring-memory-cli/tests/multi_agent_acceptance.rs new file mode 100644 index 0000000..b3db84d --- /dev/null +++ b/crates/tree-ring-memory-cli/tests/multi_agent_acceptance.rs @@ -0,0 +1,328 @@ +use std::{ + path::Path, + process::{Child, Command, Output, Stdio}, + thread, + time::{Duration, Instant}, +}; + +use serde_json::Value; +use tempfile::tempdir; +use tree_ring_memory_core::MemoryEvent; +use tree_ring_memory_sqlite::SQLiteMemoryStore; + +const WORKER_COUNT: usize = 8; +const PROJECT: &str = "multi-agent-acceptance"; +const WORKFLOW: &str = "fanout-acceptance"; +const SESSION: &str = "attempt-1"; +const QUERY_TOKEN: &str = "swarmtoken"; + +#[derive(Debug, Clone)] +struct WriteSpec { + summary: String, + scope: String, + agent_profile: String, + workflow_id: String, + session_id: String, + operation_id: String, +} + +impl WriteSpec { + fn worker(index: usize) -> Self { + Self { + summary: format!("{QUERY_TOKEN} worker {index} completed its fan-out task."), + scope: "agent".to_string(), + agent_profile: format!("worker-{index}"), + workflow_id: WORKFLOW.to_string(), + session_id: SESSION.to_string(), + operation_id: format!("operation-worker-{index}"), + } + } +} + +#[test] +fn real_cli_processes_preserve_multi_agent_isolation_and_idempotency() { + let temp = tempdir().unwrap(); + let root = temp.path().join(".tree-ring"); + + let init = base_command(&root).arg("init").output().unwrap(); + assert_success("init", &init); + let init_json = parse_json("init", &init); + assert_eq!(init_json["ok"], true); + + let worker_specs = (0..WORKER_COUNT).map(WriteSpec::worker).collect::>(); + let lock_holder = SQLiteMemoryStore::open(root.join("memory.sqlite")).unwrap(); + lock_holder + .connection() + .execute_batch("BEGIN IMMEDIATE") + .unwrap(); + let mut workers = worker_specs + .iter() + .map(|spec| { + let child = spawn_remember(&root, spec); + (spec, child) + }) + .collect::>(); + + thread::sleep(Duration::from_millis(250)); + let waiting_statuses = workers + .iter_mut() + .map(|(spec, child)| (spec.agent_profile.clone(), child.try_wait())) + .collect::>(); + lock_holder.connection().execute_batch("COMMIT").unwrap(); + drop(lock_holder); + for (agent_profile, status) in waiting_statuses { + assert!( + status.unwrap().is_none(), + "{} exited instead of waiting for the shared write lock", + agent_profile + ); + } + + let mut worker_memories = Vec::with_capacity(WORKER_COUNT); + for (spec, child) in workers.drain(..) { + let output = wait_with_output_bounded(child, &spec.agent_profile); + assert_success(&format!("remember {}", spec.agent_profile), &output); + let memory = parse_memory(&format!("remember {}", spec.agent_profile), &output); + assert_eq!(memory.scope, "agent"); + assert_eq!( + memory.agent_profile.as_deref(), + Some(spec.agent_profile.as_str()) + ); + assert_eq!( + memory.workflow_id.as_deref(), + Some(spec.workflow_id.as_str()) + ); + assert_eq!(memory.session_id.as_deref(), Some(spec.session_id.as_str())); + assert_eq!( + memory.operation_id.as_deref(), + Some(spec.operation_id.as_str()) + ); + worker_memories.push(memory); + } + + let target_profile = worker_specs[3].agent_profile.clone(); + let decoys = [ + WriteSpec { + summary: format!("{QUERY_TOKEN} workflow decoy."), + scope: "agent".to_string(), + agent_profile: target_profile.clone(), + workflow_id: "other-workflow".to_string(), + session_id: SESSION.to_string(), + operation_id: "operation-workflow-decoy".to_string(), + }, + WriteSpec { + summary: format!("{QUERY_TOKEN} session decoy."), + scope: "agent".to_string(), + agent_profile: target_profile.clone(), + workflow_id: WORKFLOW.to_string(), + session_id: "other-session".to_string(), + operation_id: "operation-session-decoy".to_string(), + }, + WriteSpec { + summary: format!("{QUERY_TOKEN} scope decoy."), + scope: "project".to_string(), + agent_profile: target_profile.clone(), + workflow_id: WORKFLOW.to_string(), + session_id: SESSION.to_string(), + operation_id: "operation-scope-decoy".to_string(), + }, + ]; + for spec in &decoys { + let output = remember_command(&root, spec).output().unwrap(); + assert_success(&format!("remember {}", spec.operation_id), &output); + parse_memory(&format!("remember {}", spec.operation_id), &output); + } + + let by_profile = recall( + &root, + &[ + ("--agent-profile", target_profile.as_str()), + ("--scope", "agent"), + ], + ); + assert_eq!(by_profile.len(), 3); + assert!(by_profile.iter().all(|memory| { + memory.agent_profile.as_deref() == Some(target_profile.as_str()) && memory.scope == "agent" + })); + + let by_workflow = recall(&root, &[("--workflow-id", WORKFLOW), ("--scope", "agent")]); + assert_eq!(by_workflow.len(), WORKER_COUNT + 1); + assert!(by_workflow + .iter() + .all(|memory| memory.workflow_id.as_deref() == Some(WORKFLOW))); + + let by_session = recall(&root, &[("--session-id", SESSION), ("--scope", "agent")]); + assert_eq!(by_session.len(), WORKER_COUNT + 1); + assert!(by_session + .iter() + .all(|memory| memory.session_id.as_deref() == Some(SESSION))); + + let by_scope = recall(&root, &[("--scope", "project")]); + assert_eq!(by_scope.len(), 1); + assert_eq!(by_scope[0].scope, "project"); + + let combined = recall( + &root, + &[ + ("--agent-profile", target_profile.as_str()), + ("--workflow-id", WORKFLOW), + ("--session-id", SESSION), + ("--scope", "agent"), + ], + ); + assert_eq!(combined.len(), 1); + assert_eq!( + combined[0].agent_profile.as_deref(), + Some(target_profile.as_str()) + ); + + let retry = remember_command(&root, &worker_specs[0]).output().unwrap(); + assert_success("idempotent operation retry", &retry); + let retried_memory = parse_memory("idempotent operation retry", &retry); + assert_eq!(retried_memory.id, worker_memories[0].id); + + let mut conflict = worker_specs[0].clone(); + conflict.summary = format!("{QUERY_TOKEN} conflicting operation reuse."); + let conflict_output = remember_command(&root, &conflict).output().unwrap(); + assert!( + !conflict_output.status.success(), + "conflicting operation unexpectedly succeeded: {}", + String::from_utf8_lossy(&conflict_output.stdout) + ); + assert!( + String::from_utf8_lossy(&conflict_output.stderr) + .contains("already bound to a different memory write"), + "unexpected conflict error: {}", + String::from_utf8_lossy(&conflict_output.stderr) + ); + + let maintain = base_command(&root).arg("maintain").output().unwrap(); + assert_success("maintenance parity report", &maintain); + let report = parse_json("maintenance parity report", &maintain); + let expected_rows = WORKER_COUNT + decoys.len(); + assert_eq!(report["memory_count"], expected_rows); + assert_eq!(report["fts"]["memory_rows"], expected_rows); + assert_eq!(report["fts"]["fts_rows"], expected_rows); + assert_eq!(report["fts"]["missing_fts_rows"], 0); + assert_eq!(report["fts"]["orphan_fts_rows"], 0); +} + +fn base_command(root: &Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_tree-ring")); + command + .env_remove("TREE_RING_AGENT_PROFILE") + .env_remove("TREE_RING_WORKFLOW_ID") + .env_remove("TREE_RING_SESSION_ID") + .arg("--root") + .arg(root) + .arg("--json"); + command +} + +fn remember_command(root: &Path, spec: &WriteSpec) -> Command { + let mut command = base_command(root); + command + .arg("remember") + .arg(&spec.summary) + .arg("--event-type") + .arg("lesson") + .arg("--scope") + .arg(&spec.scope) + .arg("--project") + .arg(PROJECT) + .arg("--agent-profile") + .arg(&spec.agent_profile) + .arg("--workflow-id") + .arg(&spec.workflow_id) + .arg("--session-id") + .arg(&spec.session_id) + .arg("--operation-id") + .arg(&spec.operation_id) + .arg("--source-ref") + .arg(format!( + "runs/{}/{}/{}.json", + spec.workflow_id, spec.session_id, spec.agent_profile + )) + .arg("--tag") + .arg("multi-agent"); + command +} + +fn spawn_remember(root: &Path, spec: &WriteSpec) -> Child { + remember_command(root, spec) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() +} + +fn wait_with_output_bounded(mut child: Child, context: &str) -> Output { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if child.try_wait().unwrap().is_some() { + return child.wait_with_output().unwrap(); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let output = child.wait_with_output().unwrap(); + panic!( + "{context} exceeded the SQLite busy-timeout bound; stdout={}; stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn recall(root: &Path, filters: &[(&str, &str)]) -> Vec { + let mut command = base_command(root); + command + .arg("recall") + .arg(QUERY_TOKEN) + .arg("--limit") + .arg("64"); + for (flag, value) in filters { + command.arg(flag).arg(value); + } + let output = command.output().unwrap(); + assert_success("recall", &output); + let payload = parse_json("recall", &output); + payload + .as_array() + .expect("recall JSON must be an array") + .iter() + .map(|entry| { + serde_json::from_value(entry["memory"].clone()) + .expect("recall entry must contain a valid memory") + }) + .collect() +} + +fn parse_memory(context: &str, output: &Output) -> MemoryEvent { + serde_json::from_slice(&output.stdout).unwrap_or_else(|error| { + panic!( + "{context} did not emit a memory JSON object: {error}; stdout={}", + String::from_utf8_lossy(&output.stdout) + ) + }) +} + +fn parse_json(context: &str, output: &Output) -> Value { + serde_json::from_slice(&output.stdout).unwrap_or_else(|error| { + panic!( + "{context} did not emit valid JSON: {error}; stdout={}", + String::from_utf8_lossy(&output.stdout) + ) + }) +} + +fn assert_success(context: &str, output: &Output) { + assert!( + output.status.success(), + "{context} failed with status {:?}; stdout={}; stderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/tree-ring-memory-core/Cargo.toml b/crates/tree-ring-memory-core/Cargo.toml index 7c8d1d0..f5611d5 100644 --- a/crates/tree-ring-memory-core/Cargo.toml +++ b/crates/tree-ring-memory-core/Cargo.toml @@ -18,6 +18,7 @@ once_cell.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true uuid.workspace = true diff --git a/crates/tree-ring-memory-core/src/audit.rs b/crates/tree-ring-memory-core/src/audit.rs index 3585bb0..ceb9ca4 100644 --- a/crates/tree-ring-memory-core/src/audit.rs +++ b/crates/tree-ring-memory-core/src/audit.rs @@ -319,6 +319,15 @@ fn audit_contradictions(events: &[MemoryEvent], findings: &mut Vec let key = ContradictionKey { project: event.project.clone(), scope: event.scope.clone(), + agent_partition: (event.scope == "agent") + .then(|| event.agent_profile.clone()) + .flatten(), + workflow_partition: (event.scope == "workflow") + .then(|| event.workflow_id.clone()) + .flatten(), + session_partition: (event.scope == "session") + .then(|| event.session_id.clone()) + .flatten(), event_type: event.event_type.clone(), tag: tag.clone(), subject: subject.clone(), @@ -358,6 +367,9 @@ fn audit_contradictions(events: &[MemoryEvent], findings: &mut Vec struct ContradictionKey { project: Option, scope: String, + agent_partition: Option, + workflow_partition: Option, + session_partition: Option, event_type: String, tag: String, subject: String, @@ -526,6 +538,89 @@ mod tests { assert_eq!(report.finding_count, 1); } + #[test] + fn private_scope_contradictions_stay_within_their_partition() { + fn directive_pair( + scope: &str, + use_partition: &str, + avoid_partition: &str, + ) -> [MemoryEvent; 2] { + let mut use_memory = MemoryEvent::new("Use shared cache policy", "decision").unwrap(); + let mut avoid_memory = + MemoryEvent::new("Avoid shared cache policy", "decision").unwrap(); + for memory in [&mut use_memory, &mut avoid_memory] { + memory.project = Some("ui".to_string()); + memory.scope = scope.to_string(); + memory.tags = vec!["cache".to_string()]; + } + match scope { + "agent" => { + use_memory.agent_profile = Some(use_partition.to_string()); + avoid_memory.agent_profile = Some(avoid_partition.to_string()); + use_memory.workflow_id = Some("workflow-use".to_string()); + avoid_memory.workflow_id = Some("workflow-avoid".to_string()); + } + "workflow" => { + use_memory.workflow_id = Some(use_partition.to_string()); + avoid_memory.workflow_id = Some(avoid_partition.to_string()); + use_memory.agent_profile = Some("coder".to_string()); + avoid_memory.agent_profile = Some("reviewer".to_string()); + } + "session" => { + use_memory.session_id = Some(use_partition.to_string()); + avoid_memory.session_id = Some(avoid_partition.to_string()); + use_memory.agent_profile = Some("coder".to_string()); + avoid_memory.agent_profile = Some("reviewer".to_string()); + } + _ => unreachable!(), + } + [use_memory, avoid_memory] + } + + for scope in ["agent", "workflow", "session"] { + let isolated = directive_pair(scope, "partition-a", "partition-b"); + let isolated_report = audit_memories(&isolated, "contradictions").unwrap(); + assert_eq!( + isolated_report.finding_count, 0, + "{scope} directives leaked across partitions" + ); + + let shared = directive_pair(scope, "partition-a", "partition-a"); + let shared_report = audit_memories(&shared, "contradictions").unwrap(); + assert_eq!( + shared_report.finding_count, 1, + "{scope} directives in one partition should be compared" + ); + } + } + + #[test] + fn shared_scope_contradictions_surface_across_producers() { + for scope in ["project", "global"] { + let mut use_memory = MemoryEvent::new("Use shared cache policy", "decision").unwrap(); + let mut avoid_memory = + MemoryEvent::new("Avoid shared cache policy", "decision").unwrap(); + for memory in [&mut use_memory, &mut avoid_memory] { + memory.project = (scope == "project").then(|| "ui".to_string()); + memory.scope = scope.to_string(); + memory.tags = vec!["cache".to_string()]; + } + use_memory.agent_profile = Some("coder".to_string()); + use_memory.workflow_id = Some("workflow-a".to_string()); + use_memory.session_id = Some("session-a".to_string()); + avoid_memory.agent_profile = Some("reviewer".to_string()); + avoid_memory.workflow_id = Some("workflow-b".to_string()); + avoid_memory.session_id = Some("session-b".to_string()); + + let report = audit_memories(&[use_memory, avoid_memory], "contradictions").unwrap(); + + assert_eq!( + report.finding_count, 1, + "{scope} directives should be compared across producers" + ); + } + } + #[test] fn audit_all_combines_checks() { let mut event = MemoryEvent::new("Private diagnosis should expire.", "lesson").unwrap(); diff --git a/crates/tree-ring-memory-core/src/consolidation.rs b/crates/tree-ring-memory-core/src/consolidation.rs index 314d4f1..3346f1e 100644 --- a/crates/tree-ring-memory-core/src/consolidation.rs +++ b/crates/tree-ring-memory-core/src/consolidation.rs @@ -71,6 +71,12 @@ pub struct ConsolidationRequest { pub period_key: Option, pub project: Option, #[serde(default)] + pub agent_profile: Option, + #[serde(default)] + pub workflow_id: Option, + #[serde(default)] + pub session_id: Option, + #[serde(default)] pub dry_run: bool, #[serde(default)] pub force: bool, @@ -82,6 +88,9 @@ impl ConsolidationRequest { period_type: ConsolidationPeriod::parse(period_type)?, period_key: None, project: None, + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }) @@ -120,6 +129,9 @@ pub struct ConsolidationReport { struct GroupKey { project: Option, scope: String, + agent_partition: Option, + workflow_partition: Option, + session_partition: Option, ring: String, event_type: String, sensitivity_bucket: String, @@ -231,6 +243,27 @@ fn is_candidate(event: &MemoryEvent, request: &ConsolidationRequest, period_key: { return false; } + if request + .agent_profile + .as_ref() + .is_some_and(|agent_profile| event.agent_profile.as_ref() != Some(agent_profile)) + { + return false; + } + if request + .workflow_id + .as_ref() + .is_some_and(|workflow_id| event.workflow_id.as_ref() != Some(workflow_id)) + { + return false; + } + if request + .session_id + .as_ref() + .is_some_and(|session_id| event.session_id.as_ref() != Some(session_id)) + { + return false; + } if effective_sensitivity(event) == "secret" { return false; } @@ -249,6 +282,15 @@ fn group_key(event: &MemoryEvent) -> GroupKey { GroupKey { project: event.project.clone(), scope: event.scope.clone(), + agent_partition: (event.scope == "agent") + .then(|| event.agent_profile.clone()) + .flatten(), + workflow_partition: (event.scope == "workflow") + .then(|| event.workflow_id.clone()) + .flatten(), + session_partition: (event.scope == "session") + .then(|| event.session_id.clone()) + .flatten(), ring: event.ring.clone(), event_type: event.event_type.clone(), sensitivity_bucket: if sensitivity == "normal" { @@ -302,6 +344,11 @@ fn build_output( }; let mut memory = MemoryEvent::new(summary, "summary")?; memory.project = key.project.clone(); + memory.agent_profile = + uniform_optional(events.iter().map(|event| event.agent_profile.as_deref())); + memory.workflow_id = uniform_optional(events.iter().map(|event| event.workflow_id.as_deref())); + memory.session_id = uniform_optional(events.iter().map(|event| event.session_id.as_deref())); + memory.operation_id = None; memory.scope = key.scope.clone(); memory.ring = output_ring(period_type, key, events).to_string(); memory.details = format!( @@ -391,6 +438,15 @@ fn average(values: impl Iterator) -> f64 { } } +fn uniform_optional<'a>(mut values: impl Iterator>) -> Option { + let first = values.next()?; + if values.all(|value| value == first) { + first.map(ToOwned::to_owned) + } else { + None + } +} + fn event_period_key(event: &MemoryEvent, period_type: ConsolidationPeriod) -> Option { let created_at = DateTime::parse_from_rfc3339(&event.created_at).ok()?; Some(period_key_for_datetime( @@ -450,6 +506,9 @@ mod tests { period_type: ConsolidationPeriod::Daily, period_key: Some("2026-07-05".to_string()), project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -481,6 +540,9 @@ mod tests { period_type: ConsolidationPeriod::Daily, period_key: Some("2026-07-05".to_string()), project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: true, force: false, }; @@ -504,6 +566,9 @@ mod tests { period_type: ConsolidationPeriod::Daily, period_key: Some("2026-07-05".to_string()), project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: true, force: false, }; @@ -522,6 +587,9 @@ mod tests { period_type: ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -544,6 +612,9 @@ mod tests { period_type: ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("private diagnosis program".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -569,6 +640,9 @@ mod tests { period_type: ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -579,4 +653,277 @@ mod tests { assert_eq!(output.tags, vec!["consolidation", "memory"]); assert!(!output.tags.iter().any(|tag| tag.contains("diagnosis"))); } + + #[test] + fn legacy_consolidation_request_defaults_private_scope_filters() { + let request: ConsolidationRequest = serde_json::from_value(serde_json::json!({ + "period_type": "manual", + "period_key": "manual-test", + "project": "ui", + "dry_run": false, + "force": false + })) + .unwrap(); + + assert_eq!(request.agent_profile, None); + assert_eq!(request.workflow_id, None); + assert_eq!(request.session_id, None); + } + + #[test] + fn agent_scoped_consolidation_isolates_producers_and_retains_correlation() { + let mut coder = event("Use the implementation plan.", "ui", "2026-07-05T08:00:00Z"); + coder.scope = "agent".to_string(); + coder.agent_profile = Some("coder".to_string()); + coder.workflow_id = Some("workflow-1".to_string()); + coder.session_id = Some("session-1".to_string()); + coder.operation_id = Some("operation-code".to_string()); + coder.tags = vec!["coordination".to_string(), "implementation".to_string()]; + + let mut reviewer = event("Review the implementation.", "ui", "2026-07-05T09:00:00Z"); + reviewer.scope = "agent".to_string(); + reviewer.agent_profile = Some("reviewer".to_string()); + reviewer.workflow_id = Some("workflow-1".to_string()); + reviewer.session_id = Some("session-1".to_string()); + reviewer.operation_id = Some("operation-review".to_string()); + reviewer.tags = vec!["coordination".to_string(), "review".to_string()]; + + let report = consolidate_memories( + &[reviewer.clone(), coder.clone()], + &ConsolidationRequest { + period_type: ConsolidationPeriod::Manual, + period_key: Some("manual-agent-isolation".to_string()), + project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + dry_run: false, + force: false, + }, + ) + .unwrap(); + + assert_eq!(report.outputs.len(), 2); + for source in [&coder, &reviewer] { + let output = report + .outputs + .iter() + .find(|output| output.memory.agent_profile == source.agent_profile) + .unwrap(); + assert_eq!(output.source_memory_ids, vec![source.id.clone()]); + assert_eq!(output.memory.workflow_id, source.workflow_id); + assert_eq!(output.memory.session_id, source.session_id); + assert_eq!(output.memory.operation_id, None); + assert!(output.memory.tags.contains(&"coordination".to_string())); + assert_eq!( + output.memory.links, + vec![MemoryLink { + link_type: "memory".to_string(), + target: source.id.clone(), + }] + ); + output.memory.validate().unwrap(); + } + } + + #[test] + fn workflow_scoped_consolidation_isolates_workflows_without_claiming_mixed_producers() { + let mut coder = event("Use the implementation plan.", "ui", "2026-07-05T08:00:00Z"); + coder.scope = "workflow".to_string(); + coder.agent_profile = Some("coder".to_string()); + coder.workflow_id = Some("workflow-1".to_string()); + coder.session_id = Some("session-shared".to_string()); + coder.operation_id = Some("operation-code".to_string()); + + let mut reviewer = event("Review the implementation.", "ui", "2026-07-05T09:00:00Z"); + reviewer.scope = "workflow".to_string(); + reviewer.agent_profile = Some("reviewer".to_string()); + reviewer.workflow_id = Some("workflow-1".to_string()); + reviewer.session_id = Some("session-shared".to_string()); + reviewer.operation_id = Some("operation-review".to_string()); + + let mut other_workflow = event("Coordinate the release.", "ui", "2026-07-05T10:00:00Z"); + other_workflow.scope = "workflow".to_string(); + other_workflow.agent_profile = Some("coordinator".to_string()); + other_workflow.workflow_id = Some("workflow-2".to_string()); + other_workflow.session_id = Some("session-other".to_string()); + other_workflow.operation_id = Some("operation-release".to_string()); + + let report = consolidate_memories( + &[coder.clone(), reviewer.clone(), other_workflow], + &ConsolidationRequest { + period_type: ConsolidationPeriod::Manual, + period_key: Some("manual-workflow-isolation".to_string()), + project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + dry_run: false, + force: false, + }, + ) + .unwrap(); + + assert_eq!(report.outputs.len(), 2); + let shared_workflow = report + .outputs + .iter() + .find(|output| output.memory.workflow_id.as_deref() == Some("workflow-1")) + .unwrap(); + assert_eq!(shared_workflow.source_memory_ids.len(), 2); + assert_eq!(shared_workflow.memory.agent_profile, None); + assert_eq!( + shared_workflow.memory.session_id.as_deref(), + Some("session-shared") + ); + assert_eq!(shared_workflow.memory.operation_id, None); + shared_workflow.memory.validate().unwrap(); + } + + #[test] + fn session_scoped_consolidation_isolates_sessions() { + let mut first = event("Use the implementation plan.", "ui", "2026-07-05T08:00:00Z"); + first.scope = "session".to_string(); + first.agent_profile = Some("coder".to_string()); + first.workflow_id = Some("workflow-1".to_string()); + first.session_id = Some("session-1".to_string()); + + let mut second = event("Review the implementation.", "ui", "2026-07-05T09:00:00Z"); + second.scope = "session".to_string(); + second.agent_profile = Some("reviewer".to_string()); + second.workflow_id = Some("workflow-2".to_string()); + second.session_id = Some("session-1".to_string()); + + let mut other_session = event("Coordinate the release.", "ui", "2026-07-05T10:00:00Z"); + other_session.scope = "session".to_string(); + other_session.agent_profile = Some("coordinator".to_string()); + other_session.workflow_id = Some("workflow-1".to_string()); + other_session.session_id = Some("session-2".to_string()); + + let report = consolidate_memories( + &[first, second, other_session], + &ConsolidationRequest { + period_type: ConsolidationPeriod::Manual, + period_key: Some("manual-session-isolation".to_string()), + project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + dry_run: false, + force: false, + }, + ) + .unwrap(); + + assert_eq!(report.outputs.len(), 2); + let shared_session = report + .outputs + .iter() + .find(|output| output.memory.session_id.as_deref() == Some("session-1")) + .unwrap(); + assert_eq!(shared_session.source_memory_ids.len(), 2); + assert_eq!(shared_session.memory.agent_profile, None); + assert_eq!(shared_session.memory.workflow_id, None); + shared_session.memory.validate().unwrap(); + } + + #[test] + fn shared_scope_consolidation_clears_mixed_producer_and_correlation_claims() { + let mut coder = event("Use the implementation plan.", "ui", "2026-07-05T08:00:00Z"); + coder.agent_profile = Some("coder".to_string()); + coder.workflow_id = Some("workflow-1".to_string()); + coder.session_id = Some("session-1".to_string()); + coder.operation_id = Some("operation-code".to_string()); + + let mut reviewer = event("Review the implementation.", "ui", "2026-07-05T09:00:00Z"); + reviewer.agent_profile = Some("reviewer".to_string()); + reviewer.workflow_id = Some("workflow-2".to_string()); + reviewer.session_id = Some("session-2".to_string()); + reviewer.operation_id = Some("operation-review".to_string()); + + let report = consolidate_memories( + &[coder, reviewer], + &ConsolidationRequest { + period_type: ConsolidationPeriod::Manual, + period_key: Some("manual-shared-producers".to_string()), + project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + dry_run: false, + force: false, + }, + ) + .unwrap(); + + assert_eq!(report.outputs.len(), 1); + let output = &report.outputs[0]; + assert_eq!(output.source_memory_ids.len(), 2); + assert_eq!(output.memory.agent_profile, None); + assert_eq!(output.memory.workflow_id, None); + assert_eq!(output.memory.session_id, None); + assert_eq!(output.memory.operation_id, None); + assert_eq!(output.memory.links.len(), 2); + } + + #[test] + fn derived_consolidation_output_does_not_reuse_a_source_operation_id() { + let mut source = event("Use the accepted plan.", "ui", "2026-07-05T08:00:00Z"); + source.operation_id = Some("operation-source".to_string()); + + let report = consolidate_memories( + &[source], + &ConsolidationRequest { + period_type: ConsolidationPeriod::Manual, + period_key: Some("manual-derived-operation".to_string()), + project: Some("ui".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, + dry_run: false, + force: false, + }, + ) + .unwrap(); + + assert_eq!(report.outputs.len(), 1); + assert_eq!(report.outputs[0].memory.operation_id, None); + } + + #[test] + fn consolidation_request_filters_each_private_scope_dimension() { + let mut matching = event("Matching memory.", "ui", "2026-07-05T08:00:00Z"); + matching.scope = "agent".to_string(); + matching.agent_profile = Some("coder".to_string()); + matching.workflow_id = Some("workflow-1".to_string()); + matching.session_id = Some("session-1".to_string()); + + let mut other_agent = matching.clone(); + other_agent.id = "mem_other_agent".to_string(); + other_agent.agent_profile = Some("reviewer".to_string()); + let mut other_workflow = matching.clone(); + other_workflow.id = "mem_other_workflow".to_string(); + other_workflow.workflow_id = Some("workflow-2".to_string()); + let mut other_session = matching.clone(); + other_session.id = "mem_other_session".to_string(); + other_session.session_id = Some("session-2".to_string()); + + let report = consolidate_memories( + &[matching.clone(), other_agent, other_workflow, other_session], + &ConsolidationRequest { + period_type: ConsolidationPeriod::Manual, + period_key: Some("manual-filtered".to_string()), + project: Some("ui".to_string()), + agent_profile: Some("coder".to_string()), + workflow_id: Some("workflow-1".to_string()), + session_id: Some("session-1".to_string()), + dry_run: false, + force: false, + }, + ) + .unwrap(); + + assert_eq!(report.candidate_count, 1); + assert_eq!(report.source_memory_ids, vec![matching.id]); + } } diff --git a/crates/tree-ring-memory-core/src/import_export.rs b/crates/tree-ring-memory-core/src/import_export.rs index df171ae..6c008cf 100644 --- a/crates/tree-ring-memory-core/src/import_export.rs +++ b/crates/tree-ring-memory-core/src/import_export.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; +use sha2::{Digest, Sha256}; use crate::models::{now_iso, MemoryEvent, TreeRingError, TreeRingResult}; use crate::sensitivity::SensitivityGuard; @@ -8,6 +9,10 @@ pub const EXPORT_RECORD_TYPE: &str = "tree_ring_memory_export"; pub const MEMORY_EVENT_RECORD_TYPE: &str = "memory_event"; pub const EXPORT_SCHEMA_VERSION: u32 = 1; pub const EXPORT_PLUGIN_VERSION: &str = env!("CARGO_PKG_VERSION"); +const LEGACY_PRIVATE_SCOPE_IDENTITY_DOMAIN: &[u8] = + b"tree-ring-memory/legacy-private-scope-identity/v1"; +const LEGACY_PRIVATE_SCOPE_REVIEW_REASON: &str = + "legacy private-scope identity synthesized during portability normalization"; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ExportHeader { @@ -89,10 +94,14 @@ pub fn decode_jsonl(input: &str) -> TreeRingResult { header = Some(parsed); } Some(MEMORY_EVENT_RECORD_TYPE) => { - let parsed: MemoryEventEnvelope = serde_json::from_value(value).map_err(|err| { - TreeRingError::Validation(format!( - "line {line_number}: invalid memory envelope: {err}" - )) + let mut parsed: MemoryEventEnvelope = + serde_json::from_value(value).map_err(|err| { + TreeRingError::Validation(format!( + "line {line_number}: invalid memory envelope: {err}" + )) + })?; + normalize_legacy_private_scope_identity(&mut parsed.memory).map_err(|err| { + TreeRingError::Validation(format!("line {line_number}: {err}")) })?; parsed.memory.validate().map_err(|err| { TreeRingError::Validation(format!("line {line_number}: {err}")) @@ -100,11 +109,14 @@ pub fn decode_jsonl(input: &str) -> TreeRingResult { events.push(parsed.memory); } _ => { - let parsed: MemoryEvent = serde_json::from_value(value).map_err(|err| { + let mut parsed: MemoryEvent = serde_json::from_value(value).map_err(|err| { TreeRingError::Validation(format!( "line {line_number}: invalid memory event: {err}" )) })?; + normalize_legacy_private_scope_identity(&mut parsed).map_err(|err| { + TreeRingError::Validation(format!("line {line_number}: {err}")) + })?; parsed.validate().map_err(|err| { TreeRingError::Validation(format!("line {line_number}: {err}")) })?; @@ -121,6 +133,7 @@ pub fn normalize_import_events(events: Vec) -> TreeRingResult TreeRingResult { + normalize_legacy_private_scope_identity(&mut event)?; let detected = SensitivityGuard::default().detect_memory_event_sensitivity(&event)?; if event.sensitivity == "normal" && detected != "normal" { event.sensitivity = detected; @@ -129,6 +142,76 @@ pub fn normalize_import_event(mut event: MemoryEvent) -> TreeRingResult TreeRingResult { + let is_missing = |value: &Option| { + value + .as_deref() + .is_none_or(|identity| identity.trim().is_empty()) + }; + let missing_identity = match event.scope.as_str() { + "agent" => is_missing(&event.agent_profile), + "workflow" => is_missing(&event.workflow_id), + "session" => is_missing(&event.session_id), + _ => false, + }; + if !missing_identity { + return Ok(false); + } + if event.id.trim().is_empty() { + return Err(TreeRingError::Validation( + "legacy private-scope normalization requires a non-blank memory id".to_string(), + )); + } + + let identity = synthetic_legacy_private_scope_identity(&event.scope, &event.id); + match event.scope.as_str() { + "agent" => event.agent_profile = Some(identity), + "workflow" => event.workflow_id = Some(identity), + "session" => event.session_id = Some(identity), + _ => unreachable!("missing_identity is only true for a private scope"), + } + event.review.needs_review = true; + event.review.reviewed_at = None; + event.review.reviewed_by = None; + match event.review.review_reason.as_mut() { + Some(reason) if !reason.contains(LEGACY_PRIVATE_SCOPE_REVIEW_REASON) => { + reason.push_str("; "); + reason.push_str(LEGACY_PRIVATE_SCOPE_REVIEW_REASON); + } + Some(_) => {} + None => { + event.review.review_reason = Some(LEGACY_PRIVATE_SCOPE_REVIEW_REASON.to_string()); + } + } + Ok(true) +} + +fn synthetic_legacy_private_scope_identity(scope: &str, memory_id: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(LEGACY_PRIVATE_SCOPE_IDENTITY_DOMAIN); + hasher.update([0]); + hasher.update(scope.as_bytes()); + hasher.update([0]); + hasher.update(memory_id.as_bytes()); + let digest = hasher.finalize(); + let compact_digest: String = digest[..16] + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + format!("legacy-{scope}-{compact_digest}") +} + #[cfg(test)] mod tests { use super::*; @@ -168,6 +251,124 @@ mod tests { assert_eq!(decoded.events, vec![event]); } + #[test] + fn decodes_and_round_trips_legacy_private_scopes_from_raw_and_enveloped_jsonl() { + let raw = include_str!("../../../fixtures/parity/legacy-private-scopes-raw.jsonl"); + let enveloped = include_str!("../../../fixtures/parity/legacy-private-scopes-export.jsonl"); + + let raw_events = decode_jsonl(raw).unwrap().events; + let enveloped_events = decode_jsonl(enveloped).unwrap().events; + + assert_eq!(raw_events, enveloped_events); + assert_eq!(raw_events.len(), 3); + for event in &raw_events { + let identity = match event.scope.as_str() { + "agent" => event.agent_profile.as_deref(), + "workflow" => event.workflow_id.as_deref(), + "session" => event.session_id.as_deref(), + scope => panic!("unexpected scope {scope}"), + } + .unwrap(); + assert!(identity.starts_with(&format!("legacy-{}-", event.scope))); + assert_eq!(identity.len(), "legacy--".len() + event.scope.len() + 32); + assert!(event.review.needs_review); + assert_eq!( + event.review.review_reason.as_deref(), + Some(LEGACY_PRIVATE_SCOPE_REVIEW_REASON) + ); + event.validate().unwrap(); + } + + let exported = encode_jsonl(&raw_events, false).unwrap(); + let round_tripped = decode_jsonl(&exported).unwrap().events; + assert_eq!(round_tripped, raw_events); + } + + #[test] + fn legacy_private_scope_normalization_is_stable_per_record_and_scope() { + let mut first = MemoryEvent::new("Legacy record one.", "lesson").unwrap(); + first.id = "mem_legacy_record_1".to_string(); + first.scope = "agent".to_string(); + let mut repeated = first.clone(); + let mut second = first.clone(); + second.id = "mem_legacy_record_2".to_string(); + let mut workflow = first.clone(); + workflow.scope = "workflow".to_string(); + + assert!(normalize_legacy_private_scope_identity(&mut first).unwrap()); + assert!(normalize_legacy_private_scope_identity(&mut repeated).unwrap()); + assert!(normalize_legacy_private_scope_identity(&mut second).unwrap()); + assert!(normalize_legacy_private_scope_identity(&mut workflow).unwrap()); + + assert_eq!(first.agent_profile, repeated.agent_profile); + assert_ne!(first.agent_profile, second.agent_profile); + assert_ne!(first.agent_profile, workflow.workflow_id); + assert!(!normalize_legacy_private_scope_identity(&mut first).unwrap()); + } + + #[test] + fn blank_legacy_private_scope_identity_is_normalized_from_fixture() { + let input = + include_str!("../../../fixtures/parity/legacy-private-scope-blank-identity.jsonl"); + + let events = decode_jsonl(input).unwrap().events; + + assert_eq!(events.len(), 1); + let event = &events[0]; + assert!(event + .agent_profile + .as_deref() + .is_some_and(|identity| identity.starts_with("legacy-agent-"))); + assert!(event.review.needs_review); + event.validate().unwrap(); + } + + #[test] + fn explicit_import_normalization_accepts_a_deserialized_legacy_record() { + let raw = include_str!("../../../fixtures/parity/legacy-private-scopes-raw.jsonl") + .lines() + .next() + .unwrap(); + let legacy: MemoryEvent = serde_json::from_str(raw).unwrap(); + + assert!(legacy + .validate() + .unwrap_err() + .to_string() + .contains("agent_profile")); + + let normalized = normalize_import_event(legacy).unwrap(); + + assert!(normalized.agent_profile.is_some()); + assert!(normalized.review.needs_review); + normalized.validate().unwrap(); + } + + #[test] + fn portability_normalization_fails_closed_for_blank_record_id() { + let mut event = MemoryEvent::new("Malformed legacy record.", "lesson").unwrap(); + event.id = " ".to_string(); + event.scope = "session".to_string(); + + let error = normalize_legacy_private_scope_identity(&mut event) + .unwrap_err() + .to_string(); + + assert!(error.contains("non-blank memory id")); + assert_eq!(event.session_id, None); + assert!(!event.review.needs_review); + } + + #[test] + fn direct_export_keeps_strict_validation_for_new_identity_less_events() { + let mut event = MemoryEvent::new("Invalid new private record.", "lesson").unwrap(); + event.scope = "workflow".to_string(); + + let error = encode_jsonl(&[event], false).unwrap_err().to_string(); + + assert!(error.contains("workflow_id")); + } + #[test] fn ignores_blank_lines() { let decoded = decode_jsonl("\n\n \n").unwrap(); diff --git a/crates/tree-ring-memory-core/src/lib.rs b/crates/tree-ring-memory-core/src/lib.rs index 57e9414..06a6404 100644 --- a/crates/tree-ring-memory-core/src/lib.rs +++ b/crates/tree-ring-memory-core/src/lib.rs @@ -17,9 +17,9 @@ pub use consolidation::{ }; pub use dox::{collect_dox_memories, DoxSyncReport, DoxSyncRequest}; pub use import_export::{ - decode_jsonl, encode_jsonl, normalize_import_event, normalize_import_events, DecodedJsonl, - ExportHeader, MemoryEventEnvelope, EXPORT_PLUGIN_VERSION, EXPORT_RECORD_TYPE, - EXPORT_SCHEMA_VERSION, MEMORY_EVENT_RECORD_TYPE, + decode_jsonl, encode_jsonl, normalize_import_event, normalize_import_events, + normalize_legacy_private_scope_identity, DecodedJsonl, ExportHeader, MemoryEventEnvelope, + EXPORT_PLUGIN_VERSION, EXPORT_RECORD_TYPE, EXPORT_SCHEMA_VERSION, MEMORY_EVENT_RECORD_TYPE, }; pub use maintenance::{ plan_maintenance, MaintenanceAction, MaintenanceActionType, MaintenanceFtsReport, diff --git a/crates/tree-ring-memory-core/src/models.rs b/crates/tree-ring-memory-core/src/models.rs index ffb6b74..b686240 100644 --- a/crates/tree-ring-memory-core/src/models.rs +++ b/crates/tree-ring-memory-core/src/models.rs @@ -23,6 +23,7 @@ pub const RETENTIONS: &[&str] = &[ "user_pinned", "forget_after_date", ]; +const MAX_CONTEXT_METADATA_LENGTH: usize = 256; pub type TreeRingResult = Result; @@ -104,6 +105,12 @@ pub struct MemoryEvent { pub project: Option, #[serde(default)] pub agent_profile: Option, + #[serde(default)] + pub workflow_id: Option, + #[serde(default)] + pub session_id: Option, + #[serde(default)] + pub operation_id: Option, #[serde(default = "default_scope")] pub scope: String, #[serde(default = "default_ring")] @@ -145,6 +152,9 @@ impl MemoryEvent { updated_at: now, project: None, agent_profile: None, + workflow_id: None, + session_id: None, + operation_id: None, scope: "global".to_string(), ring: "cambium".to_string(), event_type: event_type.into(), @@ -180,6 +190,29 @@ impl MemoryEvent { )); } validate_member("scope", &self.scope, SCOPES)?; + validate_optional_context_metadata("project", self.project.as_deref())?; + validate_optional_context_metadata("agent_profile", self.agent_profile.as_deref())?; + validate_optional_context_metadata("workflow_id", self.workflow_id.as_deref())?; + validate_optional_context_metadata("session_id", self.session_id.as_deref())?; + validate_optional_context_metadata("operation_id", self.operation_id.as_deref())?; + match self.scope.as_str() { + "agent" if self.agent_profile.is_none() => { + return Err(TreeRingError::Validation( + "scope agent requires agent_profile".to_string(), + )); + } + "workflow" if self.workflow_id.is_none() => { + return Err(TreeRingError::Validation( + "scope workflow requires workflow_id".to_string(), + )); + } + "session" if self.session_id.is_none() => { + return Err(TreeRingError::Validation( + "scope session requires session_id".to_string(), + )); + } + _ => {} + } validate_member("ring", &self.ring, RINGS)?; validate_member("sensitivity", &self.sensitivity, SENSITIVITIES)?; validate_member("retention", &self.retention, RETENTIONS)?; @@ -193,6 +226,12 @@ impl MemoryEvent { self.details.clear(); self.project = None; self.agent_profile = None; + self.workflow_id = None; + self.session_id = None; + self.operation_id = None; + if matches!(self.scope.as_str(), "agent" | "workflow" | "session") { + self.scope = "manual".to_string(); + } self.event_type = "redacted".to_string(); self.tags.clear(); self.source = MemorySource::default(); @@ -214,6 +253,28 @@ fn validate_member(field: &str, value: &str, allowed: &[&str]) -> TreeRingResult ))) } +fn validate_optional_context_metadata(field: &str, value: Option<&str>) -> TreeRingResult<()> { + let Some(value) = value else { + return Ok(()); + }; + if value.trim().is_empty() { + return Err(TreeRingError::Validation(format!( + "{field} must not be blank" + ))); + } + if value.chars().any(char::is_control) { + return Err(TreeRingError::Validation(format!( + "{field} must not contain control characters" + ))); + } + if value.chars().count() > MAX_CONTEXT_METADATA_LENGTH { + return Err(TreeRingError::Validation(format!( + "{field} must be at most {MAX_CONTEXT_METADATA_LENGTH} characters" + ))); + } + Ok(()) +} + fn default_source_type() -> String { "manual".to_string() } @@ -291,6 +352,9 @@ mod tests { assert_eq!(event.supersedes, Vec::::new()); assert_eq!(event.links, Vec::::new()); assert!(!event.review.needs_review); + assert_eq!(event.workflow_id, None); + assert_eq!(event.session_id, None); + assert_eq!(event.operation_id, None); } #[test] @@ -327,6 +391,10 @@ mod tests { fn redact_clears_sensitive_metadata() { let mut event = MemoryEvent::new("Secret memory.", "lesson").unwrap(); event.project = Some("secret".to_string()); + event.agent_profile = Some("secret".to_string()); + event.workflow_id = Some("secret".to_string()); + event.session_id = Some("secret".to_string()); + event.operation_id = Some("secret".to_string()); event.source.ref_ = "secret".to_string(); event.tags = vec!["secret".to_string()]; event.superseded_by = Some("secret".to_string()); @@ -336,6 +404,100 @@ mod tests { assert_eq!(event.summary, "[REDACTED]"); assert_eq!(event.sensitivity, "private"); + assert_eq!(event.project, None); + assert_eq!(event.agent_profile, None); + assert_eq!(event.workflow_id, None); + assert_eq!(event.session_id, None); + assert_eq!(event.operation_id, None); assert!(!serialized.contains("secret")); } + + #[test] + fn redact_moves_private_scopes_to_a_valid_identifier_free_scope() { + for scope in ["agent", "workflow", "session"] { + let mut event = MemoryEvent::new("Private scoped memory.", "lesson").unwrap(); + event.scope = scope.to_string(); + event.agent_profile = Some("worker-1".to_string()); + event.workflow_id = Some("workflow-1".to_string()); + event.session_id = Some("session-1".to_string()); + + event.redact(); + + assert_eq!(event.scope, "manual"); + assert_eq!(event.agent_profile, None); + assert_eq!(event.workflow_id, None); + assert_eq!(event.session_id, None); + event.validate().unwrap(); + } + } + + #[test] + fn private_scopes_require_their_partition_identity() { + for (scope, expected_field) in [ + ("agent", "agent_profile"), + ("workflow", "workflow_id"), + ("session", "session_id"), + ] { + let mut event = MemoryEvent::new("Scoped memory.", "decision").unwrap(); + event.scope = scope.to_string(); + + let error = event.validate().unwrap_err().to_string(); + + assert!( + error.contains(expected_field), + "expected {expected_field} error, got {error}" + ); + } + + let mut project_event = MemoryEvent::new("Project-local memory.", "decision").unwrap(); + project_event.scope = "project".to_string(); + project_event.validate().unwrap(); + } + + #[test] + fn context_metadata_rejects_blank_control_and_oversized_values() { + type Setter = fn(&mut MemoryEvent, Option); + let setters: [(&str, Setter); 5] = [ + ("project", |event, value| event.project = value), + ("agent_profile", |event, value| event.agent_profile = value), + ("workflow_id", |event, value| event.workflow_id = value), + ("session_id", |event, value| event.session_id = value), + ("operation_id", |event, value| event.operation_id = value), + ]; + let invalid_values = [ + " ".to_string(), + "value\nwith-control".to_string(), + "x".repeat(MAX_CONTEXT_METADATA_LENGTH + 1), + ]; + + for (field, set) in setters { + for invalid_value in &invalid_values { + let mut event = MemoryEvent::new("Validate context.", "decision").unwrap(); + set(&mut event, Some(invalid_value.clone())); + + let error = event.validate().unwrap_err().to_string(); + + assert!(error.contains(field), "expected {field} error, got {error}"); + } + } + } + + #[test] + fn valid_private_scope_identities_and_correlation_metadata_are_accepted() { + let mut agent = MemoryEvent::new("Agent memory.", "decision").unwrap(); + agent.scope = "agent".to_string(); + agent.agent_profile = Some("reviewer".to_string()); + agent.workflow_id = Some("workflow-42".to_string()); + agent.session_id = Some("session-7".to_string()); + agent.operation_id = Some("operation-3".to_string()); + agent.validate().unwrap(); + + let mut workflow = agent.clone(); + workflow.scope = "workflow".to_string(); + workflow.validate().unwrap(); + + let mut session = agent; + session.scope = "session".to_string(); + session.validate().unwrap(); + } } diff --git a/crates/tree-ring-memory-core/src/sensitivity.rs b/crates/tree-ring-memory-core/src/sensitivity.rs index 2eea4b8..76efa79 100644 --- a/crates/tree-ring-memory-core/src/sensitivity.rs +++ b/crates/tree-ring-memory-core/src/sensitivity.rs @@ -99,6 +99,9 @@ impl SensitivityGuard { self.accumulate(&mut detected, &event.updated_at)?; self.accumulate_optional(&mut detected, event.project.as_deref())?; self.accumulate_optional(&mut detected, event.agent_profile.as_deref())?; + self.accumulate_optional(&mut detected, event.workflow_id.as_deref())?; + self.accumulate_optional(&mut detected, event.session_id.as_deref())?; + self.accumulate_optional(&mut detected, event.operation_id.as_deref())?; self.accumulate(&mut detected, &event.scope)?; self.accumulate(&mut detected, &event.ring)?; self.accumulate(&mut detected, &event.event_type)?; @@ -276,4 +279,28 @@ mod tests { assert!(error.to_string().contains("blocked")); } + + #[test] + fn memory_event_detection_checks_all_correlation_metadata() { + let guard = SensitivityGuard::default(); + + for set_sensitive_field in [ + (|event: &mut MemoryEvent| { + event.workflow_id = Some("private diagnosis workflow".to_string()) + }) as fn(&mut MemoryEvent), + |event: &mut MemoryEvent| { + event.session_id = Some("private diagnosis session".to_string()) + }, + |event: &mut MemoryEvent| { + event.operation_id = Some("private diagnosis operation".to_string()) + }, + ] { + let mut event = MemoryEvent::new("Correlation policy.", "lesson").unwrap(); + set_sensitive_field(&mut event); + + let detected = guard.detect_memory_event_sensitivity(&event).unwrap(); + + assert_eq!(detected, "health"); + } + } } diff --git a/crates/tree-ring-memory-core/src/workflow.rs b/crates/tree-ring-memory-core/src/workflow.rs index dd5a66e..bff8161 100644 --- a/crates/tree-ring-memory-core/src/workflow.rs +++ b/crates/tree-ring-memory-core/src/workflow.rs @@ -55,6 +55,12 @@ struct WorkflowSeedMemory { project: Option, #[serde(default)] agent_profile: Option, + #[serde(default)] + workflow_id: Option, + #[serde(default)] + session_id: Option, + #[serde(default)] + operation_id: Option, #[serde(default = "default_workflow_seed_scope")] scope: String, #[serde(default = "default_workflow_seed_ring")] @@ -95,6 +101,9 @@ impl From for MemoryEvent { updated_at: seed_memory.updated_at, project: seed_memory.project, agent_profile: seed_memory.agent_profile, + workflow_id: seed_memory.workflow_id, + session_id: seed_memory.session_id, + operation_id: seed_memory.operation_id, scope: seed_memory.scope, ring: seed_memory.ring, event_type: seed_memory.event_type, diff --git a/crates/tree-ring-memory-core/tests/workflow_scenario.rs b/crates/tree-ring-memory-core/tests/workflow_scenario.rs index def06b1..17406bd 100644 --- a/crates/tree-ring-memory-core/tests/workflow_scenario.rs +++ b/crates/tree-ring-memory-core/tests/workflow_scenario.rs @@ -267,6 +267,25 @@ fn rejects_invalid_seed_memories() { assert!(parse_workflow_scenario(&input).is_err()); } +#[test] +fn workflow_seed_conversion_preserves_multi_agent_context() { + let mut seed = valid_seed_memory(); + seed["agent_profile"] = json!("reviewer"); + seed["workflow_id"] = json!("workflow-42"); + seed["session_id"] = json!("session-7"); + seed["operation_id"] = json!("operation-3"); + seed["scope"] = json!("workflow"); + + let scenario = parse_workflow_scenario(&scenario_with_seed_memory(seed)).unwrap(); + let memory = &scenario.seed_memories[0]; + + assert_eq!(memory.agent_profile.as_deref(), Some("reviewer")); + assert_eq!(memory.workflow_id.as_deref(), Some("workflow-42")); + assert_eq!(memory.session_id.as_deref(), Some("session-7")); + assert_eq!(memory.operation_id.as_deref(), Some("operation-3")); + memory.validate().unwrap(); +} + #[test] fn rejects_blank_and_cross_sensitivity_duplicate_seed_memory_ids() { let mut blank_id = valid_seed_memory(); diff --git a/crates/tree-ring-memory-sqlite/Cargo.toml b/crates/tree-ring-memory-sqlite/Cargo.toml index 0f7d3c6..d3c0faf 100644 --- a/crates/tree-ring-memory-sqlite/Cargo.toml +++ b/crates/tree-ring-memory-sqlite/Cargo.toml @@ -14,6 +14,7 @@ categories = ["database-implementations", "development-tools"] [dependencies] rusqlite.workspace = true serde_json.workspace = true +sha2.workspace = true tree-ring-memory-core = { path = "../tree-ring-memory-core", version = "0.12.0" } [dev-dependencies] diff --git a/crates/tree-ring-memory-sqlite/src/lib.rs b/crates/tree-ring-memory-sqlite/src/lib.rs index 3ca3b66..116341c 100644 --- a/crates/tree-ring-memory-sqlite/src/lib.rs +++ b/crates/tree-ring-memory-sqlite/src/lib.rs @@ -1,4 +1,7 @@ -use rusqlite::{params, params_from_iter, types::Value, Connection, ErrorCode, OptionalExtension}; +use rusqlite::{ + params, params_from_iter, types::Value, Connection, ErrorCode, OptionalExtension, + TransactionBehavior, +}; use std::collections::HashSet; use std::path::Path; @@ -6,8 +9,9 @@ use tree_ring_memory_core::models::{sqlite_error, MemoryEvent, TreeRingError, Tr use tree_ring_memory_core::recall::{search_queries, RecallScorer}; use tree_ring_memory_core::{ audit_memories, consolidate_memories, decode_jsonl, encode_jsonl, normalize_import_events, - plan_maintenance, AuditReport, ConsolidationReport, ConsolidationRequest, - MaintenanceActionType, MaintenanceFtsReport, MaintenanceReport, MaintenanceRequest, + normalize_legacy_private_scope_identity, plan_maintenance, AuditReport, ConsolidationReport, + ConsolidationRequest, MaintenanceActionType, MaintenanceFtsReport, MaintenanceReport, + MaintenanceRequest, }; mod lifecycle; @@ -15,7 +19,7 @@ mod schema; mod search; mod write; -const EXISTING_ID_QUERY_CHUNK: usize = 500; +const SQLITE_SCHEMA_VERSION: i64 = 2; #[derive(Debug, Clone)] pub struct RecallResult { @@ -24,10 +28,50 @@ pub struct RecallResult { pub ranking: std::collections::BTreeMap, } +#[derive(Debug, Clone, Copy)] +pub struct RecallOptions<'a> { + pub project: Option<&'a str>, + pub agent_profile: Option<&'a str>, + pub workflow_id: Option<&'a str>, + pub session_id: Option<&'a str>, + pub scope: Option<&'a str>, + pub rings: Option<&'a [String]>, + pub event_types: Option<&'a [String]>, + pub include_sensitive: bool, + pub include_superseded: bool, + pub limit: usize, + pub explain_ranking: bool, +} + +impl Default for RecallOptions<'_> { + fn default() -> Self { + Self { + project: None, + agent_profile: None, + workflow_id: None, + session_id: None, + scope: None, + rings: None, + event_types: None, + include_sensitive: false, + include_superseded: false, + limit: 8, + explain_ranking: false, + } + } +} + pub struct SQLiteMemoryStore { connection: Connection, } +#[derive(Debug, Clone, PartialEq)] +#[allow(clippy::large_enum_variant)] +pub enum PutOutcome { + Created, + Existing(MemoryEvent), +} + #[derive(Debug, Clone, PartialEq, Eq)] struct StoredConsolidation { id: String, @@ -69,15 +113,31 @@ impl SQLiteMemoryStore { } pub fn migrate(&self) -> TreeRingResult<()> { - self.connection - .execute_batch( - r#" + if schema::user_version(&self.connection)? >= SQLITE_SCHEMA_VERSION { + return Ok(()); + } + write::retry_locked(|| { + let transaction = rusqlite::Transaction::new_unchecked( + &self.connection, + TransactionBehavior::Immediate, + ) + .map_err(sqlite_error_from_rusqlite)?; + if schema::user_version(&transaction)? >= SQLITE_SCHEMA_VERSION { + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + return Ok(()); + } + transaction + .execute_batch( + r#" CREATE TABLE IF NOT EXISTS memories ( id TEXT PRIMARY KEY, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, project TEXT, agent_profile TEXT, + workflow_id TEXT, + session_id TEXT, + operation_id TEXT, scope TEXT NOT NULL, ring TEXT NOT NULL, event_type TEXT NOT NULL, @@ -96,6 +156,14 @@ impl SQLiteMemoryStore { review_json TEXT NOT NULL, raw_json TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS operation_claims ( + namespace_hash BLOB PRIMARY KEY NOT NULL + CHECK(length(namespace_hash) = 32), + memory_id TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS redaction_tombstones ( + memory_id TEXT PRIMARY KEY NOT NULL + ); CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5( id UNINDEXED, summary, @@ -114,9 +182,57 @@ impl SQLiteMemoryStore { notes TEXT NOT NULL ); "#, - ) - .map_err(sqlite_error_from_rusqlite)?; - Ok(()) + ) + .map_err(sqlite_error_from_rusqlite)?; + for (column, definition) in [ + ("workflow_id", "TEXT"), + ("session_id", "TEXT"), + ("operation_id", "TEXT"), + ] { + if !schema::memory_column_exists(&transaction, column)? { + transaction + .execute( + &format!("ALTER TABLE memories ADD COLUMN {column} {definition}"), + [], + ) + .map_err(sqlite_error_from_rusqlite)?; + } + } + normalize_legacy_private_scope_rows(&transaction)?; + transaction + .execute_batch( + r#" + CREATE INDEX IF NOT EXISTS idx_memories_project + ON memories(project); + CREATE INDEX IF NOT EXISTS idx_memories_agent_profile + ON memories(agent_profile); + CREATE INDEX IF NOT EXISTS idx_memories_workflow_id + ON memories(workflow_id); + CREATE INDEX IF NOT EXISTS idx_memories_session_id + ON memories(session_id); + CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_operation_namespace + ON memories( + COALESCE(project, ''), + COALESCE(workflow_id, ''), + COALESCE(agent_profile, ''), + operation_id + ) + WHERE operation_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_operation_claims_memory_id + ON operation_claims(memory_id); + INSERT OR IGNORE INTO redaction_tombstones (memory_id) + SELECT id + FROM memories + WHERE summary = '[REDACTED]'; + "#, + ) + .map_err(sqlite_error_from_rusqlite)?; + transaction + .pragma_update(None, "user_version", SQLITE_SCHEMA_VERSION) + .map_err(sqlite_error_from_rusqlite)?; + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + Ok(()) + }) } pub fn put(&mut self, event: &MemoryEvent) -> TreeRingResult<()> { @@ -131,6 +247,29 @@ impl SQLiteMemoryStore { }) } + pub fn put_idempotent(&mut self, event: &MemoryEvent) -> TreeRingResult { + let Some(operation_id) = event.operation_id.as_deref() else { + self.put(event)?; + return Ok(PutOutcome::Created); + }; + event.validate()?; + write::retry_locked(|| { + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(sqlite_error_from_rusqlite)?; + if let Some(existing) = + find_memory_by_operation_namespace(&transaction, event, operation_id)? + { + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + return Ok(PutOutcome::Existing(existing)); + } + write::put_in_transaction(&transaction, event)?; + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + Ok(PutOutcome::Created) + }) + } + pub fn put_many(&mut self, events: &[MemoryEvent]) -> TreeRingResult<()> { write::retry_locked(|| { let transaction = self @@ -139,16 +278,7 @@ impl SQLiteMemoryStore { .map_err(sqlite_error_from_rusqlite)?; { let mut insert_memory = transaction - .prepare( - r#" - INSERT OR REPLACE INTO memories ( - id, created_at, updated_at, project, agent_profile, scope, ring, - event_type, summary, details, source_json, tags_json, salience, - confidence, sensitivity, retention, expires_at, supersedes_json, - superseded_by, links_json, review_json, raw_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - "#, - ) + .prepare(write::UPSERT_MEMORY_SQL) .map_err(sqlite_error_from_rusqlite)?; let mut delete_fts = transaction .prepare("DELETE FROM memory_fts WHERE id = ?") @@ -158,6 +288,7 @@ impl SQLiteMemoryStore { .map_err(sqlite_error_from_rusqlite)?; for event in events { + write::prepare_memory_write(&transaction, event)?; write::put_with_statements( event, &mut insert_memory, @@ -184,19 +315,7 @@ impl SQLiteMemoryStore { } pub fn list_all(&self, include_superseded: bool) -> TreeRingResult> { - let sql = if include_superseded { - "SELECT raw_json FROM memories ORDER BY created_at DESC" - } else { - "SELECT raw_json FROM memories WHERE superseded_by IS NULL ORDER BY created_at DESC" - }; - let mut statement = self - .connection - .prepare(sql) - .map_err(sqlite_error_from_rusqlite)?; - let rows = statement - .query_map([], search::event_from_row) - .map_err(sqlite_error_from_rusqlite)?; - search::collect_rows(rows) + list_all_on_connection(&self.connection, include_superseded) } pub fn search_text( @@ -269,6 +388,36 @@ impl SQLiteMemoryStore { include_sensitive: bool, include_superseded: bool, limit: Option, + ) -> TreeRingResult> { + self.search_text_filtered_with_context_limited( + query, + project, + agent_profile, + None, + None, + scope, + rings, + event_types, + include_sensitive, + include_superseded, + limit, + ) + } + + #[allow(clippy::too_many_arguments)] + fn search_text_filtered_with_context_limited( + &self, + query: &str, + project: Option<&str>, + agent_profile: Option<&str>, + workflow_id: Option<&str>, + session_id: Option<&str>, + scope: Option<&str>, + rings: Option<&[String]>, + event_types: Option<&[String]>, + include_sensitive: bool, + include_superseded: bool, + limit: Option, ) -> TreeRingResult> { if query.trim().is_empty() { return Ok(Vec::new()); @@ -285,6 +434,8 @@ impl SQLiteMemoryStore { &fts_query, project, agent_profile, + workflow_id, + session_id, scope, rings, event_types, @@ -300,6 +451,8 @@ impl SQLiteMemoryStore { fts_query: &str, project: Option<&str>, agent_profile: Option<&str>, + workflow_id: Option<&str>, + session_id: Option<&str>, scope: Option<&str>, rings: Option<&[String]>, event_types: Option<&[String]>, @@ -328,6 +481,14 @@ impl SQLiteMemoryStore { sql.push_str(" AND memories.agent_profile = ?"); parameters.push(Value::Text(agent_profile.to_string())); } + if let Some(workflow_id) = workflow_id { + sql.push_str(" AND memories.workflow_id = ?"); + parameters.push(Value::Text(workflow_id.to_string())); + } + if let Some(session_id) = session_id { + sql.push_str(" AND memories.session_id = ?"); + parameters.push(Value::Text(session_id.to_string())); + } if let Some(scope) = scope { sql.push_str(" AND memories.scope = ?"); parameters.push(Value::Text(scope.to_string())); @@ -363,18 +524,13 @@ impl SQLiteMemoryStore { } pub fn supersede(&mut self, old_id: &str, new_id: &str) -> TreeRingResult<()> { - let Some(mut old) = self.get(old_id)? else { - return Ok(()); - }; - old.superseded_by = Some(new_id.to_string()); - let raw_json = serde_json::to_string(&old)?; write::retry_locked(|| { - self.connection - .execute( - "UPDATE memories SET superseded_by = ?, raw_json = ? WHERE id = ?", - params![new_id, raw_json, old_id], - ) + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(sqlite_error_from_rusqlite)?; + write::supersede_in_transaction(&transaction, old_id, new_id)?; + transaction.commit().map_err(sqlite_error_from_rusqlite)?; Ok(()) }) } @@ -385,23 +541,61 @@ impl SQLiteMemoryStore { .connection .transaction() .map_err(sqlite_error_from_rusqlite)?; - transaction - .execute("DELETE FROM memories WHERE id = ?", params![memory_id]) - .map_err(sqlite_error_from_rusqlite)?; - transaction - .execute("DELETE FROM memory_fts WHERE id = ?", params![memory_id]) - .map_err(sqlite_error_from_rusqlite)?; + write::delete_in_transaction(&transaction, memory_id)?; transaction.commit().map_err(sqlite_error_from_rusqlite)?; Ok(()) }) } pub fn redact(&mut self, memory_id: &str) -> TreeRingResult<()> { - let Some(mut event) = self.get(memory_id)? else { - return Ok(()); - }; - event.redact(); - self.put(&event) + write::retry_locked(|| { + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(sqlite_error_from_rusqlite)?; + write::redact_in_transaction(&transaction, memory_id)?; + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + Ok(()) + }) + } + + pub fn change_ring( + &mut self, + memory_id: &str, + ring: &str, + event_type: &str, + ) -> TreeRingResult> { + write::retry_locked(|| { + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(sqlite_error_from_rusqlite)?; + let Some(mut event) = transaction + .query_row( + "SELECT raw_json FROM memories WHERE id = ?", + params![memory_id], + search::event_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose()? + else { + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + return Ok(None); + }; + let tombstoned = write::is_redaction_tombstoned(&transaction, memory_id)?; + event.ring = ring.to_string(); + if !tombstoned { + event.event_type = event_type.to_string(); + } + event.updated_at = tree_ring_memory_core::now_iso(); + if ring == "heartwood" { + event.retention = "durable".to_string(); + } + write::put_in_transaction(&transaction, &event)?; + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + Ok(Some(event)) + }) } pub fn export_jsonl( @@ -442,32 +636,47 @@ impl SQLiteMemoryStore { return Ok(report); } - let ids = events - .iter() - .map(|event| event.id.clone()) - .collect::>(); - let mut known_ids = self.existing_memory_ids(&ids)?; - let mut batch_events = Vec::new(); - for event in events { - if known_ids.contains(&event.id) { - if replace_existing { - batch_events.push(event); - report.replaced_count += 1; - } else { - report.skipped_duplicate_count += 1; + let (inserted_count, replaced_count, skipped_duplicate_count) = + write::retry_locked(|| { + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(sqlite_error_from_rusqlite)?; + let mut inserted_count = 0; + let mut replaced_count = 0; + let mut skipped_duplicate_count = 0; + let mut written_events = Vec::new(); + for event in &events { + let exists: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM memories WHERE id = ?)", + params![&event.id], + |row| row.get(0), + ) + .map_err(sqlite_error_from_rusqlite)?; + if exists && !replace_existing { + skipped_duplicate_count += 1; + continue; + } + write::put_in_transaction(&transaction, event)?; + written_events.push(event); + if exists { + replaced_count += 1; + } else { + inserted_count += 1; + } } - } else { - known_ids.insert(event.id.clone()); - batch_events.push(event); - report.inserted_count += 1; - } - } - if !batch_events.is_empty() { - self.put_many(&batch_events)?; - for event in &batch_events { - self.apply_supersedes(event)?; - } - } + for event in written_events { + for old_id in &event.supersedes { + write::supersede_in_transaction(&transaction, old_id, &event.id)?; + } + } + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + Ok((inserted_count, replaced_count, skipped_duplicate_count)) + })?; + report.inserted_count = inserted_count; + report.replaced_count = replaced_count; + report.skipped_duplicate_count = skipped_duplicate_count; Ok(report) } @@ -480,292 +689,130 @@ impl SQLiteMemoryStore { &mut self, request: &ConsolidationRequest, ) -> TreeRingResult { - let events = self.list_all(false)?; - let mut report = consolidate_memories(&events, request)?; - if request.dry_run || report.candidate_count == 0 { - return Ok(report); + if request.dry_run { + let events = self.list_all(false)?; + return consolidate_memories(&events, request); } - - let source_ids_json = - serde_json::to_string(&report.source_memory_ids).map_err(TreeRingError::Json)?; - if !request.force { - if let Some(existing) = self.find_consolidation( - report.period_type.as_str(), - &report.period_key, - &source_ids_json, - )? { - report.id = existing.id; - report.created_at = existing.created_at; - report.output_memory_ids = existing.output_memory_ids; - report.status = "unchanged".to_string(); - report.notes = "Matching consolidation already exists.".to_string(); - report.outputs.clear(); + write::retry_locked(|| { + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(sqlite_error_from_rusqlite)?; + let events = list_all_on_connection(&transaction, false)?; + let mut report = consolidate_memories(&events, request)?; + if report.candidate_count == 0 { + transaction.commit().map_err(sqlite_error_from_rusqlite)?; return Ok(report); } - } - let previous_outputs = if request.force { - self.previous_consolidation_outputs(report.period_type.as_str(), &report.period_key)? - } else { - Vec::new() - }; - let output_events = report - .outputs - .iter() - .map(|output| output.memory.clone()) - .collect::>(); - let supersession_pairs = if request.force { - consolidation_supersession_pairs(&previous_outputs, &output_events) - } else { - Vec::new() - }; - report.status = "created".to_string(); - report.notes = "Consolidation summaries stored.".to_string(); - self.insert_consolidation_transaction(&report, &output_events, &supersession_pairs)?; - Ok(report) + let source_ids_json = + serde_json::to_string(&report.source_memory_ids).map_err(TreeRingError::Json)?; + if !request.force { + if let Some(existing) = find_consolidation_on_connection( + &transaction, + report.period_type.as_str(), + &report.period_key, + &source_ids_json, + )? { + report.id = existing.id; + report.created_at = existing.created_at; + report.output_memory_ids = existing.output_memory_ids; + report.status = "unchanged".to_string(); + report.notes = "Matching consolidation already exists.".to_string(); + report.outputs.clear(); + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + return Ok(report); + } + } + + let output_events = report + .outputs + .iter() + .map(|output| output.memory.clone()) + .collect::>(); + let supersession_pairs = if request.force { + let previous_outputs = previous_consolidation_outputs_on_connection( + &transaction, + report.period_type.as_str(), + &report.period_key, + request, + )?; + consolidation_supersession_pairs(&previous_outputs, &output_events) + } else { + Vec::new() + }; + report.status = "created".to_string(); + report.notes = "Consolidation summaries stored.".to_string(); + for event in &output_events { + write::put_in_transaction(&transaction, event)?; + } + for (old_id, new_id) in &supersession_pairs { + write::supersede_in_transaction(&transaction, old_id, new_id)?; + } + insert_consolidation_record(&transaction, &report, &source_ids_json)?; + transaction.commit().map_err(sqlite_error_from_rusqlite)?; + Ok(report) + }) } pub fn maintain(&mut self, request: &MaintenanceRequest) -> TreeRingResult { - let events = self.list_all(true)?; - let mut report = plan_maintenance(&events, request); - report.fts = self.fts_report(false)?; - let apply_expired = !request.dry_run && request.apply_expired; let apply_secret_redactions = !request.dry_run && request.apply_secret_redactions; let repair_fts = !request.dry_run && request.repair_fts; - let needs_transaction = repair_fts - || report.actions.iter().any(|action| { - matches!(action.action_type, MaintenanceActionType::DeleteExpired) && apply_expired - || matches!(action.action_type, MaintenanceActionType::RedactSecret) - && apply_secret_redactions - }); - if needs_transaction { - let (applied_indexes, fts_repaired) = write::retry_locked(|| { + let needs_transaction = apply_expired || apply_secret_redactions || repair_fts; + let mut report = if needs_transaction { + write::retry_locked(|| { let transaction = self .connection - .transaction() + .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(sqlite_error_from_rusqlite)?; - let mut transaction_applied = Vec::new(); + let events = list_all_on_connection(&transaction, true)?; + let mut transaction_report = plan_maintenance(&events, request); + transaction_report.fts = fts_report_on_connection(&transaction, false)?; - for (index, action) in report.actions.iter().enumerate() { + for action in &mut transaction_report.actions { if action.action_type == MaintenanceActionType::RedactSecret && apply_secret_redactions && write::redact_in_transaction(&transaction, &action.memory_id)? { - transaction_applied.push(index); + action.applied = true; } } - for (index, action) in report.actions.iter().enumerate() { + for action in &mut transaction_report.actions { if action.action_type == MaintenanceActionType::DeleteExpired && apply_expired && write::delete_in_transaction(&transaction, &action.memory_id)? { - transaction_applied.push(index); + action.applied = true; } } if repair_fts { lifecycle::rebuild_fts_in_transaction(&transaction)?; } - + transaction_report.applied_action_count = transaction_report + .actions + .iter() + .filter(|action| action.applied) + .count(); + transaction_report.fts = fts_report_on_connection(&transaction, repair_fts)?; transaction.commit().map_err(sqlite_error_from_rusqlite)?; - Ok((transaction_applied, repair_fts)) - })?; - - for index in applied_indexes { - if let Some(action) = report.actions.get_mut(index) { - action.applied = true; - } - } - report.applied_action_count = report - .actions - .iter() - .filter(|action| action.applied) - .count(); - report.fts = self.fts_report(fts_repaired)?; - } + Ok(transaction_report) + })? + } else { + let events = self.list_all(true)?; + let mut report = plan_maintenance(&events, request); + report.fts = self.fts_report(false)?; + report + }; report.status = maintenance_status(&report); Ok(report) } - fn find_consolidation( - &self, - period_type: &str, - period_key: &str, - source_ids_json: &str, - ) -> TreeRingResult> { - self.connection - .query_row( - r#" - SELECT id, created_at, output_memory_ids_json - FROM consolidations - WHERE period_type = ? - AND period_key = ? - AND source_memory_ids_json = ? - AND status = 'created' - ORDER BY created_at DESC - LIMIT 1 - "#, - params![period_type, period_key, source_ids_json], - lifecycle::stored_consolidation_from_row, - ) - .optional() - .map_err(sqlite_error_from_rusqlite)? - .transpose() - } - - fn previous_consolidation_outputs( - &self, - period_type: &str, - period_key: &str, - ) -> TreeRingResult> { - let mut statement = self - .connection - .prepare( - r#" - SELECT output_memory_ids_json - FROM consolidations - WHERE period_type = ? - AND period_key = ? - AND status = 'created' - ORDER BY created_at ASC - "#, - ) - .map_err(sqlite_error_from_rusqlite)?; - let rows = statement - .query_map(params![period_type, period_key], |row| { - row.get::<_, String>(0) - }) - .map_err(sqlite_error_from_rusqlite)?; - let mut output_ids = Vec::new(); - for row in rows { - let output_ids_json = row.map_err(sqlite_error_from_rusqlite)?; - let mut parsed = serde_json::from_str::>(&output_ids_json) - .map_err(TreeRingError::Json)?; - output_ids.append(&mut parsed); - } - let mut outputs = Vec::new(); - for output_id in output_ids { - if let Some(output) = self.get(&output_id)? { - outputs.push(output); - } - } - Ok(outputs) - } - - fn insert_consolidation_transaction( - &mut self, - report: &ConsolidationReport, - output_events: &[MemoryEvent], - supersession_pairs: &[(MemoryEvent, String)], - ) -> TreeRingResult<()> { - let source_ids_json = - serde_json::to_string(&report.source_memory_ids).map_err(TreeRingError::Json)?; - let output_ids_json = - serde_json::to_string(&report.output_memory_ids).map_err(TreeRingError::Json)?; - write::retry_locked(|| { - let transaction = self - .connection - .transaction() - .map_err(sqlite_error_from_rusqlite)?; - for event in output_events { - write::put_in_transaction(&transaction, event)?; - } - for (old, new_id) in supersession_pairs { - let mut updated = old.clone(); - updated.superseded_by = Some(new_id.clone()); - let raw_json = serde_json::to_string(&updated)?; - transaction - .execute( - "UPDATE memories SET superseded_by = ?, raw_json = ? WHERE id = ?", - params![new_id, raw_json, &old.id], - ) - .map_err(sqlite_error_from_rusqlite)?; - } - transaction - .execute( - r#" - INSERT INTO consolidations ( - id, created_at, period_type, period_key, source_memory_ids_json, - output_memory_ids_json, status, notes - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - "#, - params![ - report.id, - report.created_at, - report.period_type.as_str(), - &report.period_key, - source_ids_json, - output_ids_json, - &report.status, - &report.notes - ], - ) - .map_err(sqlite_error_from_rusqlite)?; - transaction.commit().map_err(sqlite_error_from_rusqlite)?; - Ok(()) - }) - } - - fn apply_supersedes(&mut self, event: &MemoryEvent) -> TreeRingResult<()> { - for old_id in &event.supersedes { - self.supersede(old_id, &event.id)?; - } - Ok(()) - } - - fn existing_memory_ids(&self, ids: &[String]) -> TreeRingResult> { - let mut existing = HashSet::new(); - for chunk in ids.chunks(EXISTING_ID_QUERY_CHUNK) { - if chunk.is_empty() { - continue; - } - let placeholders = std::iter::repeat_n("?", chunk.len()) - .collect::>() - .join(", "); - let sql = format!("SELECT id FROM memories WHERE id IN ({placeholders})"); - let mut statement = self - .connection - .prepare(&sql) - .map_err(sqlite_error_from_rusqlite)?; - let rows = statement - .query_map(params_from_iter(chunk.iter()), |row| { - row.get::<_, String>(0) - }) - .map_err(sqlite_error_from_rusqlite)?; - for row in rows { - existing.insert(row.map_err(sqlite_error_from_rusqlite)?); - } - } - Ok(existing) - } - fn fts_report(&self, repaired: bool) -> TreeRingResult { - Ok(MaintenanceFtsReport { - memory_rows: lifecycle::count_query(&self.connection, "SELECT count(*) FROM memories")?, - fts_rows: lifecycle::count_query(&self.connection, "SELECT count(*) FROM memory_fts")?, - missing_fts_rows: lifecycle::count_query( - &self.connection, - r#" - SELECT count(*) - FROM memories - LEFT JOIN memory_fts ON memories.id = memory_fts.id - WHERE memory_fts.id IS NULL - "#, - )?, - orphan_fts_rows: lifecycle::count_query( - &self.connection, - r#" - SELECT count(*) - FROM memory_fts - LEFT JOIN memories ON memories.id = memory_fts.id - WHERE memories.id IS NULL - "#, - )?, - repaired, - }) + fts_report_on_connection(&self.connection, repaired) } } @@ -791,6 +838,29 @@ impl<'a> MemoryRetriever<'a> { include_superseded: bool, limit: usize, explain_ranking: bool, + ) -> TreeRingResult> { + self.recall_with_options( + query, + &RecallOptions { + project, + agent_profile, + workflow_id: None, + session_id: None, + scope, + rings, + event_types, + include_sensitive, + include_superseded, + limit, + explain_ranking, + }, + ) + } + + pub fn recall_with_options( + &self, + query: &str, + options: &RecallOptions<'_>, ) -> TreeRingResult> { if query.trim().is_empty() { return Ok(Vec::new()); @@ -798,20 +868,22 @@ impl<'a> MemoryRetriever<'a> { let mut candidates = Vec::new(); let mut seen_queries = HashSet::new(); - let candidate_limit = Some(limit.saturating_mul(128).clamp(256, 2048)); + let candidate_limit = Some(options.limit.saturating_mul(128).clamp(256, 2048)); for search_query in search_queries(query) { if !seen_queries.insert(search_query.clone()) { continue; } - candidates = self.store.search_text_filtered_limited( + candidates = self.store.search_text_filtered_with_context_limited( &search_query, - project, - agent_profile, - scope, - rings, - event_types, - include_sensitive, - include_superseded, + options.project, + options.agent_profile, + options.workflow_id, + options.session_id, + options.scope, + options.rings, + options.event_types, + options.include_sensitive, + options.include_superseded, candidate_limit, )?; if !candidates.is_empty() { @@ -822,13 +894,15 @@ impl<'a> MemoryRetriever<'a> { if let Some(fts_query) = format_plain_text_fts_or_query(query) { candidates = self.store.search_fts_filtered_limited( &fts_query, - project, - agent_profile, - scope, - rings, - event_types, - include_sensitive, - include_superseded, + options.project, + options.agent_profile, + options.workflow_id, + options.session_id, + options.scope, + options.rings, + options.event_types, + options.include_sensitive, + options.include_superseded, candidate_limit, )?; } @@ -839,12 +913,14 @@ impl<'a> MemoryRetriever<'a> { .filter(|event| { matches_filters( event, - project, - agent_profile, - scope, - rings, - event_types, - include_sensitive, + options.project, + options.agent_profile, + options.workflow_id, + options.session_id, + options.scope, + options.rings, + options.event_types, + options.include_sensitive, ) }) .map(|memory| { @@ -852,7 +928,7 @@ impl<'a> MemoryRetriever<'a> { RecallResult { memory, score: scored.score, - ranking: if explain_ranking { + ranking: if options.explain_ranking { scored.ranking.factors } else { Default::default() @@ -861,15 +937,18 @@ impl<'a> MemoryRetriever<'a> { }) .collect(); results.sort_by(|left, right| right.score.total_cmp(&left.score)); - results.truncate(limit); + results.truncate(options.limit); Ok(results) } } +#[allow(clippy::too_many_arguments)] fn matches_filters( event: &MemoryEvent, project: Option<&str>, agent_profile: Option<&str>, + workflow_id: Option<&str>, + session_id: Option<&str>, scope: Option<&str>, rings: Option<&[String]>, event_types: Option<&[String]>, @@ -881,6 +960,12 @@ fn matches_filters( if agent_profile.is_some_and(|profile| event.agent_profile.as_deref() != Some(profile)) { return false; } + if workflow_id.is_some_and(|workflow| event.workflow_id.as_deref() != Some(workflow)) { + return false; + } + if session_id.is_some_and(|session| event.session_id.as_deref() != Some(session)) { + return false; + } if scope.is_some_and(|scope| event.scope != scope) { return false; } @@ -899,7 +984,7 @@ fn matches_filters( fn consolidation_supersession_pairs( previous_outputs: &[MemoryEvent], new_outputs: &[MemoryEvent], -) -> Vec<(MemoryEvent, String)> { +) -> Vec<(String, String)> { if new_outputs.is_empty() { return Vec::new(); } @@ -909,11 +994,233 @@ fn consolidation_supersession_pairs( .map(|(index, old)| { let target = best_consolidation_replacement(old, new_outputs) .unwrap_or_else(|| &new_outputs[index % new_outputs.len()]); - (old.clone(), target.id.clone()) + (old.id.clone(), target.id.clone()) }) .collect() } +fn find_memory_by_operation_namespace( + connection: &Connection, + event: &MemoryEvent, + operation_id: &str, +) -> TreeRingResult> { + let active = connection + .query_row( + r#" + SELECT raw_json + FROM memories + WHERE COALESCE(project, '') = COALESCE(?, '') + AND COALESCE(workflow_id, '') = COALESCE(?, '') + AND COALESCE(agent_profile, '') = COALESCE(?, '') + AND operation_id = ? + LIMIT 1 + "#, + params![ + event.project.as_deref(), + event.workflow_id.as_deref(), + event.agent_profile.as_deref(), + operation_id + ], + search::event_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose()?; + if active.is_some() { + return Ok(active); + } + let claim_hash = write::operation_namespace_hash(event, operation_id); + connection + .query_row( + r#" + SELECT memories.raw_json + FROM operation_claims + JOIN memories ON memories.id = operation_claims.memory_id + WHERE operation_claims.namespace_hash = ? + LIMIT 1 + "#, + params![claim_hash.as_slice()], + search::event_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose() +} + +fn list_all_on_connection( + connection: &Connection, + include_superseded: bool, +) -> TreeRingResult> { + let sql = if include_superseded { + "SELECT raw_json FROM memories ORDER BY created_at DESC" + } else { + "SELECT raw_json FROM memories WHERE superseded_by IS NULL ORDER BY created_at DESC" + }; + let mut statement = connection + .prepare(sql) + .map_err(sqlite_error_from_rusqlite)?; + let rows = statement + .query_map([], search::event_from_row) + .map_err(sqlite_error_from_rusqlite)?; + search::collect_rows(rows) +} + +fn fts_report_on_connection( + connection: &Connection, + repaired: bool, +) -> TreeRingResult { + Ok(MaintenanceFtsReport { + memory_rows: lifecycle::count_query(connection, "SELECT count(*) FROM memories")?, + fts_rows: lifecycle::count_query(connection, "SELECT count(*) FROM memory_fts")?, + missing_fts_rows: lifecycle::count_query( + connection, + r#" + SELECT count(*) + FROM memories + LEFT JOIN memory_fts ON memories.id = memory_fts.id + WHERE memory_fts.id IS NULL + "#, + )?, + orphan_fts_rows: lifecycle::count_query( + connection, + r#" + SELECT count(*) + FROM memory_fts + LEFT JOIN memories ON memories.id = memory_fts.id + WHERE memories.id IS NULL + "#, + )?, + repaired, + }) +} + +fn previous_consolidation_outputs_on_connection( + connection: &Connection, + period_type: &str, + period_key: &str, + request: &ConsolidationRequest, +) -> TreeRingResult> { + let mut statement = connection + .prepare( + r#" + SELECT output_memory_ids_json + FROM consolidations + WHERE period_type = ? + AND period_key = ? + AND status = 'created' + ORDER BY created_at ASC + "#, + ) + .map_err(sqlite_error_from_rusqlite)?; + let rows = statement + .query_map(params![period_type, period_key], |row| { + row.get::<_, String>(0) + }) + .map_err(sqlite_error_from_rusqlite)?; + let mut output_ids = Vec::new(); + for row in rows { + let output_ids_json = row.map_err(sqlite_error_from_rusqlite)?; + let mut parsed = + serde_json::from_str::>(&output_ids_json).map_err(TreeRingError::Json)?; + output_ids.append(&mut parsed); + } + drop(statement); + + let mut outputs = Vec::new(); + for output_id in output_ids { + if let Some(output) = connection + .query_row( + "SELECT raw_json FROM memories WHERE id = ?", + params![output_id], + search::event_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose()? + { + if consolidation_context_matches(&output, request) { + outputs.push(output); + } + } + } + Ok(outputs) +} + +fn consolidation_context_matches(event: &MemoryEvent, request: &ConsolidationRequest) -> bool { + request + .project + .as_ref() + .is_none_or(|project| event.project.as_ref() == Some(project)) + && request + .agent_profile + .as_ref() + .is_none_or(|profile| event.agent_profile.as_ref() == Some(profile)) + && request + .workflow_id + .as_ref() + .is_none_or(|workflow| event.workflow_id.as_ref() == Some(workflow)) + && request + .session_id + .as_ref() + .is_none_or(|session| event.session_id.as_ref() == Some(session)) +} + +fn insert_consolidation_record( + connection: &Connection, + report: &ConsolidationReport, + source_ids_json: &str, +) -> TreeRingResult<()> { + let output_ids_json = + serde_json::to_string(&report.output_memory_ids).map_err(TreeRingError::Json)?; + connection + .execute( + r#" + INSERT INTO consolidations ( + id, created_at, period_type, period_key, source_memory_ids_json, + output_memory_ids_json, status, notes + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + "#, + params![ + report.id, + report.created_at, + report.period_type.as_str(), + &report.period_key, + source_ids_json, + output_ids_json, + &report.status, + &report.notes + ], + ) + .map_err(sqlite_error_from_rusqlite)?; + Ok(()) +} + +fn find_consolidation_on_connection( + connection: &Connection, + period_type: &str, + period_key: &str, + source_ids_json: &str, +) -> TreeRingResult> { + connection + .query_row( + r#" + SELECT id, created_at, output_memory_ids_json + FROM consolidations + WHERE period_type = ? + AND period_key = ? + AND source_memory_ids_json = ? + AND status = 'created' + ORDER BY created_at DESC + LIMIT 1 + "#, + params![period_type, period_key, source_ids_json], + lifecycle::stored_consolidation_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose() +} + fn best_consolidation_replacement<'a>( old: &MemoryEvent, new_outputs: &'a [MemoryEvent], @@ -957,6 +1264,56 @@ fn maintenance_status(report: &MaintenanceReport) -> String { } } +fn normalize_legacy_private_scope_rows( + transaction: &rusqlite::Transaction<'_>, +) -> TreeRingResult<()> { + let rows = { + let mut statement = transaction + .prepare( + r#" + SELECT raw_json + FROM memories + WHERE scope IN ('agent', 'workflow', 'session') + "#, + ) + .map_err(sqlite_error_from_rusqlite)?; + let mapped = statement + .query_map([], |row| row.get::<_, String>(0)) + .map_err(sqlite_error_from_rusqlite)?; + mapped + .collect::, _>>() + .map_err(sqlite_error_from_rusqlite)? + }; + + for raw_json in rows { + let mut event: MemoryEvent = serde_json::from_str(&raw_json)?; + normalize_legacy_private_scope_identity(&mut event)?; + event.validate()?; + transaction + .execute( + r#" + UPDATE memories + SET agent_profile = ?, + workflow_id = ?, + session_id = ?, + review_json = ?, + raw_json = ? + WHERE id = ? + "#, + params![ + event.agent_profile.as_deref(), + event.workflow_id.as_deref(), + event.session_id.as_deref(), + serde_json::to_string(&event.review)?, + serde_json::to_string(&event)?, + &event.id, + ], + ) + .map_err(sqlite_error_from_rusqlite)?; + } + Ok(()) +} + fn sqlite_error_from_rusqlite(error: rusqlite::Error) -> TreeRingError { let is_locked = matches!( &error, @@ -998,7 +1355,10 @@ const SEARCH_FILLER_TERMS: &[&str] = &[ #[cfg(test)] mod tests { use super::*; - use std::thread; + use std::{ + sync::{Arc, Barrier}, + thread, + }; use tempfile::tempdir; use tree_ring_memory_core::models::MemorySource; @@ -1071,6 +1431,221 @@ mod tests { assert!(busy_timeout >= 30_000); } + #[test] + fn read_only_store_sees_committed_rows_in_live_wal() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut writer = SQLiteMemoryStore::open(&db_path).unwrap(); + writer + .connection() + .execute_batch("PRAGMA wal_autocheckpoint=0; PRAGMA wal_checkpoint(TRUNCATE);") + .unwrap(); + let event = MemoryEvent::new("Visible from the live WAL.", "lesson").unwrap(); + + writer.put(&event).unwrap(); + + let wal_path = db_path.with_extension("sqlite-wal"); + assert!(std::fs::metadata(&wal_path).unwrap().len() > 0); + let reader = SQLiteMemoryStore::open_read_only(&db_path).unwrap(); + assert_eq!(reader.get(&event.id).unwrap().unwrap().id, event.id); + let query_only: i64 = reader + .connection() + .query_row("PRAGMA query_only", [], |row| row.get(0)) + .unwrap(); + assert_eq!(query_only, 1); + assert!(reader + .connection() + .execute("CREATE TABLE forbidden_write (id INTEGER)", []) + .is_err()); + } + + #[test] + fn migrate_adds_multi_agent_columns_and_indexes_to_legacy_database() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + create_legacy_database(&db_path); + + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + + for column in ["workflow_id", "session_id", "operation_id"] { + assert!(schema::memory_column_exists(store.connection(), column).unwrap()); + } + let indexes: HashSet = { + let mut statement = store + .connection() + .prepare("SELECT name FROM sqlite_master WHERE type = 'index'") + .unwrap(); + statement + .query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .map(Result::unwrap) + .collect() + }; + for index in [ + "idx_memories_project", + "idx_memories_agent_profile", + "idx_memories_workflow_id", + "idx_memories_session_id", + "idx_memories_operation_namespace", + ] { + assert!(indexes.contains(index), "missing index {index}"); + } + } + + #[test] + fn migration_normalizes_and_round_trips_identity_less_legacy_private_scope() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + create_legacy_database(&db_path); + let mut legacy = + MemoryEvent::new("Legacy workflow memory remains portable.", "lesson").unwrap(); + legacy.id = "mem_legacy_workflow_portability".to_string(); + legacy.scope = "workflow".to_string(); + insert_legacy_event(&db_path, &legacy); + + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + let migrated = store.get(&legacy.id).unwrap().unwrap(); + let workflow_id = migrated.workflow_id.as_deref().unwrap(); + assert!(workflow_id.starts_with("legacy-workflow-")); + assert!(migrated.review.needs_review); + let stored_workflow_id: String = store + .connection() + .query_row( + "SELECT workflow_id FROM memories WHERE id = ?", + params![&legacy.id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored_workflow_id, workflow_id); + + let (jsonl, report) = store.export_jsonl(false, false).unwrap(); + assert_eq!(report.memory_count, 1); + let target_dir = tempdir().unwrap(); + let mut target = SQLiteMemoryStore::open(target_dir.path().join("memory.sqlite")).unwrap(); + assert_eq!( + target + .import_jsonl(&jsonl, false, false) + .unwrap() + .inserted_count, + 1 + ); + assert_eq!( + target + .get(&legacy.id) + .unwrap() + .unwrap() + .workflow_id + .as_deref(), + Some(workflow_id) + ); + } + + #[test] + fn migration_normalizes_blank_legacy_agent_identity() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + create_legacy_database(&db_path); + let mut legacy = + MemoryEvent::new("Legacy blank agent identity is normalized.", "lesson").unwrap(); + legacy.id = "mem_legacy_blank_agent_migration".to_string(); + legacy.scope = "agent".to_string(); + legacy.agent_profile = Some(" \t ".to_string()); + insert_legacy_event(&db_path, &legacy); + + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + let migrated = store.get(&legacy.id).unwrap().unwrap(); + + assert!(migrated + .agent_profile + .as_deref() + .is_some_and(|identity| identity.starts_with("legacy-agent-"))); + assert!(migrated.review.needs_review); + migrated.validate().unwrap(); + } + + #[test] + fn read_only_legacy_rows_are_normalized_in_memory_without_migration() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + create_legacy_database(&db_path); + let mut legacy = MemoryEvent::new( + "Legacy session memory is readable without mutation.", + "lesson", + ) + .unwrap(); + legacy.id = "mem_legacy_read_only_session".to_string(); + legacy.scope = "session".to_string(); + insert_legacy_event(&db_path, &legacy); + + let store = SQLiteMemoryStore::open_read_only(&db_path).unwrap(); + let loaded = store.get(&legacy.id).unwrap().unwrap(); + + assert!(loaded + .session_id + .as_deref() + .is_some_and(|identity| identity.starts_with("legacy-session-"))); + assert!(loaded.review.needs_review); + assert!(store.export_jsonl(false, false).is_ok()); + assert!(!schema::memory_column_exists(store.connection(), "session_id").unwrap()); + } + + #[test] + fn concurrent_open_serializes_legacy_schema_migration() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + create_legacy_database(&db_path); + let barrier = Arc::new(Barrier::new(3)); + let handles: Vec<_> = (0..2) + .map(|_| { + let db_path = db_path.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + barrier.wait(); + SQLiteMemoryStore::open(db_path) + }) + }) + .collect(); + + barrier.wait(); + for handle in handles { + handle.join().unwrap().unwrap(); + } + + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + for column in ["workflow_id", "session_id", "operation_id"] { + assert!(schema::memory_column_exists(store.connection(), column).unwrap()); + } + } + + #[test] + fn current_schema_open_does_not_wait_for_writer_lock() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut writer = SQLiteMemoryStore::open(&db_path).unwrap(); + assert_eq!( + schema::user_version(writer.connection()).unwrap(), + SQLITE_SCHEMA_VERSION + ); + let transaction = writer + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let (result_tx, result_rx) = std::sync::mpsc::channel(); + let handle = thread::spawn(move || { + let result = SQLiteMemoryStore::open(&db_path) + .and_then(|store| schema::user_version(store.connection())); + result_tx.send(result).unwrap(); + }); + + let opened_version = result_rx + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("current-schema open waited for the writer lock") + .unwrap(); + assert_eq!(opened_version, SQLITE_SCHEMA_VERSION); + transaction.commit().unwrap(); + handle.join().unwrap(); + } + #[test] fn store_searches_fts() { let dir = tempdir().unwrap(); @@ -1102,12 +1677,93 @@ mod tests { .connection() .query_row("SELECT count(*) FROM memories", [], |row| row.get(0)) .unwrap(); - let fts_count: i64 = store - .connection() - .query_row("SELECT count(*) FROM memory_fts", [], |row| row.get(0)) - .unwrap(); - assert_eq!(memory_count, 5); - assert_eq!(fts_count, 5); + let fts_count: i64 = store + .connection() + .query_row("SELECT count(*) FROM memory_fts", [], |row| row.get(0)) + .unwrap(); + assert_eq!(memory_count, 5); + assert_eq!(fts_count, 5); + } + + #[test] + fn put_idempotent_returns_existing_without_replacing_a_different_id() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut first = MemoryEvent::new("First operation result.", "lesson").unwrap(); + first.project = Some("core".to_string()); + first.workflow_id = Some("workflow-1".to_string()); + first.agent_profile = Some("agent-a".to_string()); + first.operation_id = Some("operation-1".to_string()); + let mut duplicate = first.clone(); + duplicate.id = "mem_duplicate_operation".to_string(); + duplicate.summary = "A retry with a different id.".to_string(); + + assert_eq!(store.put_idempotent(&first).unwrap(), PutOutcome::Created); + let outcome = store.put_idempotent(&duplicate).unwrap(); + + match outcome { + PutOutcome::Existing(existing) => assert_eq!(existing.id, first.id), + PutOutcome::Created => panic!("duplicate operation unexpectedly created a row"), + } + assert!(store.get(&duplicate.id).unwrap().is_none()); + assert!(store.put(&duplicate).is_err()); + assert_eq!( + store.get(&first.id).unwrap().unwrap().summary, + "First operation result." + ); + } + + #[test] + fn concurrent_put_idempotent_claims_operation_once() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + SQLiteMemoryStore::open(&db_path).unwrap(); + let barrier = Arc::new(Barrier::new(3)); + let handles: Vec<_> = (0..2) + .map(|index| { + let db_path = db_path.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + let mut event = + MemoryEvent::new(format!("Concurrent operation {index}."), "lesson") + .unwrap(); + event.project = Some("core".to_string()); + event.workflow_id = Some("workflow-1".to_string()); + event.agent_profile = Some("agent-a".to_string()); + event.operation_id = Some("operation-1".to_string()); + let mut store = SQLiteMemoryStore::open(db_path).unwrap(); + barrier.wait(); + store.put_idempotent(&event).unwrap() + }) + }) + .collect(); + + barrier.wait(); + let outcomes: Vec<_> = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect(); + + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PutOutcome::Created)) + .count(), + 1 + ); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, PutOutcome::Existing(_))) + .count(), + 1 + ); + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + let rows: i64 = store + .connection() + .query_row("SELECT count(*) FROM memories", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, 1); } #[test] @@ -1184,6 +1840,62 @@ mod tests { assert_eq!(results[0].memory.id, target_id); } + #[test] + fn recall_filters_workflow_and_session_before_candidate_limit() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut events = Vec::new(); + for index in 0..300 { + let mut event = MemoryEvent::new( + format!("Workflow partition bottleneck distractor {index}"), + "lesson", + ) + .unwrap(); + event.workflow_id = Some("other-workflow".to_string()); + event.session_id = Some("target-session".to_string()); + events.push(event); + } + let mut workflow_target = + MemoryEvent::new("Workflow partition bottleneck target", "lesson").unwrap(); + workflow_target.workflow_id = Some("target-workflow".to_string()); + workflow_target.session_id = Some("target-session".to_string()); + let workflow_target_id = workflow_target.id.clone(); + events.push(workflow_target); + for index in 0..300 { + let mut event = MemoryEvent::new( + format!("Session partition bottleneck distractor {index}"), + "lesson", + ) + .unwrap(); + event.workflow_id = Some("target-workflow".to_string()); + event.session_id = Some("other-session".to_string()); + events.push(event); + } + let mut session_target = + MemoryEvent::new("Session partition bottleneck target", "lesson").unwrap(); + session_target.workflow_id = Some("target-workflow".to_string()); + session_target.session_id = Some("target-session".to_string()); + let session_target_id = session_target.id.clone(); + events.push(session_target); + store.put_many(&events).unwrap(); + let options = RecallOptions { + workflow_id: Some("target-workflow"), + session_id: Some("target-session"), + limit: 1, + ..RecallOptions::default() + }; + + let workflow_results = MemoryRetriever::new(&store) + .recall_with_options("workflow partition bottleneck", &options) + .unwrap(); + let session_results = MemoryRetriever::new(&store) + .recall_with_options("session partition bottleneck", &options) + .unwrap(); + + assert_eq!(workflow_results[0].memory.id, workflow_target_id); + assert_eq!(session_results[0].memory.id, session_target_id); + } + #[test] fn recall_strict_all_term_match_wins_before_fallback() { let dir = tempdir().unwrap(); @@ -1321,6 +2033,262 @@ mod tests { .is_empty()); } + #[test] + fn redaction_preserves_operation_claim_against_exact_retry() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut event = MemoryEvent::new("Payload that must stay forgotten.", "lesson").unwrap(); + event.project = Some("core".to_string()); + event.workflow_id = Some("workflow-1".to_string()); + event.agent_profile = Some("agent-a".to_string()); + event.operation_id = Some("operation-1".to_string()); + assert_eq!(store.put_idempotent(&event).unwrap(), PutOutcome::Created); + + store.redact(&event.id).unwrap(); + let mut retry = event.clone(); + retry.id = "mem_exact_retry_after_redaction".to_string(); + let outcome = store.put_idempotent(&retry).unwrap(); + + match outcome { + PutOutcome::Existing(existing) => { + assert_eq!(existing.id, event.id); + assert_eq!(existing.summary, "[REDACTED]"); + assert!(existing.operation_id.is_none()); + } + PutOutcome::Created => panic!("redacted operation claim was unexpectedly released"), + } + assert!(store.get(&retry.id).unwrap().is_none()); + assert!(store.put(&retry).is_err()); + let scrubbed_columns: [Option; 5] = store + .connection() + .query_row( + r#" + SELECT project, agent_profile, workflow_id, session_id, operation_id + FROM memories + WHERE id = ? + "#, + params![&event.id], + |row| { + Ok([ + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + ]) + }, + ) + .unwrap(); + assert_eq!(scrubbed_columns, [None, None, None, None, None]); + let (claim_hash, claim_memory_id): (Vec, String) = store + .connection() + .query_row( + "SELECT namespace_hash, memory_id FROM operation_claims", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(claim_hash.len(), 32); + assert_eq!(claim_memory_id, event.id); + assert!(!claim_hash + .windows("operation-1".len()) + .any(|window| window == b"operation-1")); + let rows: i64 = store + .connection() + .query_row("SELECT count(*) FROM memories", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, 1); + } + + #[test] + fn redacted_id_cannot_be_resurrected_by_put_or_replacement_import() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let original = MemoryEvent::new( + "Payload that replacement import must not restore.", + "lesson", + ) + .unwrap(); + store.put(&original).unwrap(); + store.redact(&original.id).unwrap(); + + let mut replacement = original.clone(); + replacement.summary = "Resurrected payload.".to_string(); + assert!(store.put(&replacement).is_err()); + + let jsonl = encode_jsonl(&[replacement], false).unwrap(); + assert!(store.import_jsonl(&jsonl, false, true).is_err()); + + let retained = store.get(&original.id).unwrap().unwrap(); + assert_eq!(retained.summary, "[REDACTED]"); + assert_eq!(retained.event_type, "redacted"); + let tombstones: i64 = store + .connection() + .query_row( + "SELECT count(*) FROM redaction_tombstones WHERE memory_id = ?", + params![&original.id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(tombstones, 1); + } + + #[test] + fn replacing_active_operation_preserves_old_namespace_until_hard_delete() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut original = MemoryEvent::new("Original operation payload.", "lesson").unwrap(); + original.project = Some("core".to_string()); + original.workflow_id = Some("workflow-1".to_string()); + original.agent_profile = Some("agent-a".to_string()); + original.operation_id = Some("operation-a".to_string()); + store.put(&original).unwrap(); + + let mut replacement = original.clone(); + replacement.summary = "Replacement operation payload.".to_string(); + replacement.operation_id = Some("operation-b".to_string()); + store.put(&replacement).unwrap(); + + let mut retry = original.clone(); + retry.id = "mem_retry_replaced_operation".to_string(); + let outcome = store.put_idempotent(&retry).unwrap(); + match outcome { + PutOutcome::Existing(existing) => { + assert_eq!(existing.id, original.id); + assert_eq!(existing.operation_id.as_deref(), Some("operation-b")); + } + PutOutcome::Created => panic!("replaced operation namespace was unexpectedly released"), + } + assert!(store.get(&retry.id).unwrap().is_none()); + + store.delete(&original.id).unwrap(); + assert_eq!(store.put_idempotent(&retry).unwrap(), PutOutcome::Created); + } + + #[test] + fn hard_delete_removes_redacted_operation_claim() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut event = MemoryEvent::new("Deleted operation payload.", "lesson").unwrap(); + event.project = Some("core".to_string()); + event.operation_id = Some("operation-delete".to_string()); + store.put(&event).unwrap(); + store.redact(&event.id).unwrap(); + + store.delete(&event.id).unwrap(); + + let claims: i64 = store + .connection() + .query_row("SELECT count(*) FROM operation_claims", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(claims, 0); + let tombstones: i64 = store + .connection() + .query_row("SELECT count(*) FROM redaction_tombstones", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(tombstones, 0); + let mut retry = event.clone(); + retry.id = "mem_retry_after_hard_delete".to_string(); + assert_eq!(store.put_idempotent(&retry).unwrap(), PutOutcome::Created); + } + + #[test] + fn concurrent_redact_and_supersede_preserve_both_monotonic_changes() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut setup = SQLiteMemoryStore::open(&db_path).unwrap(); + let event = MemoryEvent::new("Secret payload must not be resurrected.", "lesson").unwrap(); + let event_id = event.id.clone(); + setup.put(&event).unwrap(); + drop(setup); + + let barrier = Arc::new(Barrier::new(3)); + let redact_handle = { + let db_path = db_path.clone(); + let event_id = event_id.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + let mut store = SQLiteMemoryStore::open(db_path).unwrap(); + barrier.wait(); + store.redact(&event_id).unwrap(); + }) + }; + let supersede_handle = { + let db_path = db_path.clone(); + let event_id = event_id.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + let mut store = SQLiteMemoryStore::open(db_path).unwrap(); + barrier.wait(); + store.supersede(&event_id, "mem_replacement").unwrap(); + }) + }; + + barrier.wait(); + redact_handle.join().unwrap(); + supersede_handle.join().unwrap(); + + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + let updated = store.get(&event_id).unwrap().unwrap(); + assert_eq!(updated.summary, "[REDACTED]"); + assert_eq!(updated.superseded_by.as_deref(), Some("mem_replacement")); + assert!(!serde_json::to_string(&updated) + .unwrap() + .contains("Secret payload")); + } + + #[test] + fn concurrent_ring_change_reloads_redacted_row_under_write_lock() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut setup = SQLiteMemoryStore::open(&db_path).unwrap(); + let event = MemoryEvent::new("Sensitive payload must stay gone.", "lesson").unwrap(); + let event_id = event.id.clone(); + setup.put(&event).unwrap(); + drop(setup); + + let barrier = Arc::new(Barrier::new(3)); + let redact_handle = { + let db_path = db_path.clone(); + let event_id = event_id.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + let mut store = SQLiteMemoryStore::open(db_path).unwrap(); + barrier.wait(); + store.redact(&event_id).unwrap(); + }) + }; + let promote_handle = { + let db_path = db_path.clone(); + let event_id = event_id.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + let mut store = SQLiteMemoryStore::open(db_path).unwrap(); + barrier.wait(); + store + .change_ring(&event_id, "heartwood", "decision") + .unwrap(); + }) + }; + + barrier.wait(); + redact_handle.join().unwrap(); + promote_handle.join().unwrap(); + + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + let updated = store.get(&event_id).unwrap().unwrap(); + assert_eq!(updated.summary, "[REDACTED]"); + assert_eq!(updated.ring, "heartwood"); + assert_eq!(updated.retention, "durable"); + assert!(!serde_json::to_string(&updated) + .unwrap() + .contains("Sensitive payload")); + } + #[test] fn export_jsonl_excludes_sensitive_and_superseded_by_default() { let dir = tempdir().unwrap(); @@ -1508,6 +2476,59 @@ mod tests { assert_eq!(active_ids, vec![new.id]); } + #[test] + fn import_rolls_back_all_rows_when_operation_namespace_conflicts() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut existing = MemoryEvent::new("Existing operation claimant.", "lesson").unwrap(); + existing.project = Some("core".to_string()); + existing.workflow_id = Some("workflow-1".to_string()); + existing.agent_profile = Some("agent-a".to_string()); + existing.operation_id = Some("operation-1".to_string()); + store.put(&existing).unwrap(); + let new_event = MemoryEvent::new("This import must roll back.", "lesson").unwrap(); + let mut conflict = existing.clone(); + conflict.id = "mem_import_operation_conflict".to_string(); + conflict.summary = "Conflicting imported operation.".to_string(); + let jsonl = encode_jsonl(&[new_event.clone(), conflict.clone()], false).unwrap(); + + assert!(store.import_jsonl(&jsonl, false, false).is_err()); + + assert!(store.get(&new_event.id).unwrap().is_none()); + assert!(store.get(&conflict.id).unwrap().is_none()); + assert_eq!( + store.get(&existing.id).unwrap().unwrap().summary, + "Existing operation claimant." + ); + } + + #[test] + fn import_cannot_bypass_redacted_operation_claim() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut redacted = MemoryEvent::new("Original operation payload.", "lesson").unwrap(); + redacted.project = Some("core".to_string()); + redacted.workflow_id = Some("workflow-1".to_string()); + redacted.agent_profile = Some("agent-a".to_string()); + redacted.operation_id = Some("operation-1".to_string()); + store.put(&redacted).unwrap(); + store.redact(&redacted.id).unwrap(); + + let first = MemoryEvent::new("Earlier import row must roll back.", "lesson").unwrap(); + let mut retry = redacted.clone(); + retry.id = "mem_import_retry_after_redaction".to_string(); + let jsonl = encode_jsonl(&[first.clone(), retry.clone()], false).unwrap(); + + assert!(store.import_jsonl(&jsonl, false, false).is_err()); + + assert!(store.get(&first.id).unwrap().is_none()); + assert!(store.get(&retry.id).unwrap().is_none()); + assert_eq!( + store.get(&redacted.id).unwrap().unwrap().summary, + "[REDACTED]" + ); + } + #[test] fn audit_uses_all_rows_and_does_not_mutate_storage() { let dir = tempdir().unwrap(); @@ -1555,6 +2576,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: true, force: false, }; @@ -1583,6 +2607,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-empty".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -1614,6 +2641,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -1632,6 +2662,111 @@ mod tests { assert_eq!(records, 1); } + #[test] + fn concurrent_consolidation_claim_creates_one_record_and_output_set() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut setup = SQLiteMemoryStore::open(&db_path).unwrap(); + let mut event = + MemoryEvent::new("Consolidate this source exactly once.", "decision").unwrap(); + event.project = Some("core".to_string()); + setup.put(&event).unwrap(); + drop(setup); + + let mut request = ConsolidationRequest::new("manual").unwrap(); + request.period_key = Some("manual-concurrent".to_string()); + request.project = Some("core".to_string()); + let barrier = Arc::new(Barrier::new(3)); + let handles: Vec<_> = (0..2) + .map(|_| { + let db_path = db_path.clone(); + let request = request.clone(); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + let mut store = SQLiteMemoryStore::open(db_path).unwrap(); + barrier.wait(); + store.consolidate(&request).unwrap() + }) + }) + .collect(); + + barrier.wait(); + let reports: Vec<_> = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect(); + + assert_eq!( + reports + .iter() + .filter(|report| report.status == "created") + .count(), + 1 + ); + assert_eq!( + reports + .iter() + .filter(|report| report.status == "unchanged") + .count(), + 1 + ); + assert_eq!(reports[0].output_memory_ids, reports[1].output_memory_ids); + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + let records: i64 = store + .connection() + .query_row("SELECT count(*) FROM consolidations", [], |row| row.get(0)) + .unwrap(); + let rows: i64 = store + .connection() + .query_row("SELECT count(*) FROM memories", [], |row| row.get(0)) + .unwrap(); + assert_eq!(records, 1); + assert_eq!(rows as usize, 1 + reports[0].output_memory_ids.len()); + } + + #[test] + fn consolidation_reloads_sources_after_waiting_for_concurrent_redaction() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut writer = SQLiteMemoryStore::open(&db_path).unwrap(); + let mut event = + MemoryEvent::new("Source being redacted before consolidation.", "lesson").unwrap(); + event.project = Some("core".to_string()); + event.tags = vec!["stale_sensitive_marker".to_string()]; + writer.put(&event).unwrap(); + let mut consolidator = SQLiteMemoryStore::open(&db_path).unwrap(); + let mut request = ConsolidationRequest::new("manual").unwrap(); + request.period_key = Some("manual-redaction-race".to_string()); + request.project = Some("core".to_string()); + + let transaction = writer + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + write::redact_in_transaction(&transaction, &event.id).unwrap(); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let handle = thread::spawn(move || { + started_tx.send(()).unwrap(); + consolidator.consolidate(&request).unwrap() + }); + started_rx.recv().unwrap(); + thread::sleep(std::time::Duration::from_millis(100)); + transaction.commit().unwrap(); + + let report = handle.join().unwrap(); + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + for output_id in report.output_memory_ids { + let output = store.get(&output_id).unwrap().unwrap(); + assert!(!output + .tags + .iter() + .any(|tag| tag == "stale_sensitive_marker")); + assert!(!serde_json::to_string(&output) + .unwrap() + .contains("stale_sensitive_marker")); + } + } + #[test] fn forced_consolidation_supersedes_prior_summary() { let dir = tempdir().unwrap(); @@ -1643,6 +2778,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -1676,6 +2814,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-test".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -1702,6 +2843,71 @@ mod tests { assert!(second.source_memory_ids.contains(&second_event.id)); } + #[test] + fn forced_consolidation_does_not_supersede_other_project_outputs() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut project_a = MemoryEvent::new("Project A decision.", "decision").unwrap(); + project_a.project = Some("project-a".to_string()); + let mut project_b = MemoryEvent::new("Project B decision.", "decision").unwrap(); + project_b.project = Some("project-b".to_string()); + store.put(&project_a).unwrap(); + store.put(&project_b).unwrap(); + let mut request_a = ConsolidationRequest::new("manual").unwrap(); + request_a.period_key = Some("manual-project-isolation".to_string()); + request_a.project = Some("project-a".to_string()); + let mut request_b = request_a.clone(); + request_b.project = Some("project-b".to_string()); + let first_a = store.consolidate(&request_a).unwrap(); + let first_b = store.consolidate(&request_b).unwrap(); + + request_a.force = true; + let second_a = store.consolidate(&request_a).unwrap(); + + let old_a = store.get(&first_a.output_memory_ids[0]).unwrap().unwrap(); + let old_b = store.get(&first_b.output_memory_ids[0]).unwrap().unwrap(); + assert_eq!( + old_a.superseded_by.as_deref(), + Some(second_a.output_memory_ids[0].as_str()) + ); + assert!(old_b.superseded_by.is_none()); + } + + #[test] + fn forced_consolidation_does_not_supersede_other_workflow_outputs() { + let dir = tempdir().unwrap(); + let mut store = SQLiteMemoryStore::open(dir.path().join("memory.sqlite")).unwrap(); + let mut workflow_a = MemoryEvent::new("Workflow A decision.", "decision").unwrap(); + workflow_a.project = Some("core".to_string()); + workflow_a.scope = "workflow".to_string(); + workflow_a.workflow_id = Some("workflow-a".to_string()); + let mut workflow_b = MemoryEvent::new("Workflow B decision.", "decision").unwrap(); + workflow_b.project = Some("core".to_string()); + workflow_b.scope = "workflow".to_string(); + workflow_b.workflow_id = Some("workflow-b".to_string()); + store.put(&workflow_a).unwrap(); + store.put(&workflow_b).unwrap(); + let mut request_a = ConsolidationRequest::new("manual").unwrap(); + request_a.period_key = Some("manual-workflow-isolation".to_string()); + request_a.project = Some("core".to_string()); + request_a.workflow_id = Some("workflow-a".to_string()); + let mut request_b = request_a.clone(); + request_b.workflow_id = Some("workflow-b".to_string()); + let first_a = store.consolidate(&request_a).unwrap(); + let first_b = store.consolidate(&request_b).unwrap(); + + request_a.force = true; + let second_a = store.consolidate(&request_a).unwrap(); + + let old_a = store.get(&first_a.output_memory_ids[0]).unwrap().unwrap(); + let old_b = store.get(&first_b.output_memory_ids[0]).unwrap().unwrap(); + assert_eq!( + old_a.superseded_by.as_deref(), + Some(second_a.output_memory_ids[0].as_str()) + ); + assert!(old_b.superseded_by.is_none()); + } + #[test] fn forced_consolidation_maps_multiple_prior_outputs_to_matching_new_outputs() { let dir = tempdir().unwrap(); @@ -1717,6 +2923,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-multi-output".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -1769,6 +2978,9 @@ mod tests { period_type: tree_ring_memory_core::ConsolidationPeriod::Manual, period_key: Some("manual-sensitive".to_string()), project: Some("core".to_string()), + agent_profile: None, + workflow_id: None, + session_id: None, dry_run: false, force: false, }; @@ -1839,6 +3051,40 @@ mod tests { assert!(!protected_action.applied); } + #[test] + fn maintenance_replans_after_waiting_for_concurrent_writer() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("memory.sqlite"); + let mut writer = SQLiteMemoryStore::open(&db_path).unwrap(); + let mut maintainer = SQLiteMemoryStore::open(&db_path).unwrap(); + let expired = + expired_maintenance_memory("Concurrent expired cache", "ephemeral", "cambium"); + let transaction = writer + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + write::put_in_transaction(&transaction, &expired).unwrap(); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let handle = thread::spawn(move || { + started_tx.send(()).unwrap(); + maintainer + .maintain(&MaintenanceRequest { + dry_run: false, + apply_expired: true, + ..MaintenanceRequest::default() + }) + .unwrap() + }); + started_rx.recv().unwrap(); + thread::sleep(std::time::Duration::from_millis(100)); + transaction.commit().unwrap(); + + let report = handle.join().unwrap(); + let store = SQLiteMemoryStore::open(&db_path).unwrap(); + assert_eq!(report.applied_action_count, 1); + assert!(store.get(&expired.id).unwrap().is_none()); + } + #[test] fn maintenance_apply_secret_redactions_redacts_secret_like_memory() { let dir = tempdir().unwrap(); @@ -2010,6 +3256,80 @@ mod tests { event } + fn create_legacy_database(path: &Path) { + let legacy = Connection::open(path).unwrap(); + legacy + .execute_batch( + r#" + CREATE TABLE memories ( + id TEXT PRIMARY KEY, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + project TEXT, + agent_profile TEXT, + scope TEXT NOT NULL, + ring TEXT NOT NULL, + event_type TEXT NOT NULL, + summary TEXT NOT NULL, + details TEXT NOT NULL, + source_json TEXT NOT NULL, + tags_json TEXT NOT NULL, + salience REAL NOT NULL, + confidence REAL NOT NULL, + sensitivity TEXT NOT NULL, + retention TEXT NOT NULL, + expires_at TEXT, + supersedes_json TEXT NOT NULL, + superseded_by TEXT, + links_json TEXT NOT NULL, + review_json TEXT NOT NULL, + raw_json TEXT NOT NULL + ); + "#, + ) + .unwrap(); + } + + fn insert_legacy_event(path: &Path, event: &MemoryEvent) { + let legacy = Connection::open(path).unwrap(); + legacy + .execute( + r#" + INSERT INTO memories ( + id, created_at, updated_at, project, agent_profile, scope, ring, + event_type, summary, details, source_json, tags_json, salience, + confidence, sensitivity, retention, expires_at, supersedes_json, + superseded_by, links_json, review_json, raw_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "#, + params![ + &event.id, + &event.created_at, + &event.updated_at, + event.project.as_deref(), + event.agent_profile.as_deref(), + &event.scope, + &event.ring, + &event.event_type, + &event.summary, + &event.details, + serde_json::to_string(&event.source).unwrap(), + serde_json::to_string(&event.tags).unwrap(), + event.salience, + event.confidence, + &event.sensitivity, + &event.retention, + event.expires_at.as_deref(), + serde_json::to_string(&event.supersedes).unwrap(), + event.superseded_by.as_deref(), + serde_json::to_string(&event.links).unwrap(), + serde_json::to_string(&event.review).unwrap(), + serde_json::to_string(event).unwrap(), + ], + ) + .unwrap(); + } + fn assert_action_type( report: &MaintenanceReport, memory_id: &str, diff --git a/crates/tree-ring-memory-sqlite/src/schema.rs b/crates/tree-ring-memory-sqlite/src/schema.rs index 6da2858..1b80a91 100644 --- a/crates/tree-ring-memory-sqlite/src/schema.rs +++ b/crates/tree-ring-memory-sqlite/src/schema.rs @@ -10,7 +10,7 @@ pub(crate) fn open_connection(path: &Path) -> TreeRingResult { std::fs::create_dir_all(parent).map_err(|err| sqlite_error(err.to_string()))?; } let connection = Connection::open(path).map_err(sqlite_error_from_rusqlite)?; - configure_connection(&connection)?; + configure_writable_connection(&connection)?; Ok(connection) } @@ -19,20 +19,17 @@ pub(crate) fn open_read_only_connection(path: &Path) -> TreeRingResult TreeRingResult<()> { +fn configure_writable_connection(connection: &Connection) -> TreeRingResult<()> { connection .busy_timeout(std::time::Duration::from_millis(30_000)) .map_err(sqlite_error_from_rusqlite)?; @@ -44,6 +41,16 @@ fn configure_connection(connection: &Connection) -> TreeRingResult<()> { Ok(()) } +fn configure_read_only_connection(connection: &Connection) -> TreeRingResult<()> { + connection + .busy_timeout(std::time::Duration::from_millis(30_000)) + .map_err(sqlite_error_from_rusqlite)?; + connection + .execute_batch("PRAGMA query_only=ON; PRAGMA busy_timeout=30000;") + .map_err(sqlite_error_from_rusqlite)?; + Ok(()) +} + pub(crate) fn parent_dir_to_create(path: &Path) -> Option<&Path> { path.parent() .filter(|parent| !parent.as_os_str().is_empty()) @@ -70,6 +77,29 @@ pub(crate) fn sqlite_uri_path(path: &str) -> String { .collect() } +pub(crate) fn memory_column_exists( + connection: &Connection, + column_name: &str, +) -> TreeRingResult { + let mut statement = connection + .prepare("PRAGMA table_info(memories)") + .map_err(sqlite_error_from_rusqlite)?; + let mut rows = statement.query([]).map_err(sqlite_error_from_rusqlite)?; + while let Some(row) = rows.next().map_err(sqlite_error_from_rusqlite)? { + let existing_name: String = row.get(1).map_err(sqlite_error_from_rusqlite)?; + if existing_name == column_name { + return Ok(true); + } + } + Ok(false) +} + +pub(crate) fn user_version(connection: &Connection) -> TreeRingResult { + connection + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(sqlite_error_from_rusqlite) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/tree-ring-memory-sqlite/src/search.rs b/crates/tree-ring-memory-sqlite/src/search.rs index 170500c..d14b084 100644 --- a/crates/tree-ring-memory-sqlite/src/search.rs +++ b/crates/tree-ring-memory-sqlite/src/search.rs @@ -1,12 +1,19 @@ use rusqlite::{types::Value, Result as SqliteResult, Row}; -use tree_ring_memory_core::models::{MemoryEvent, TreeRingResult}; +use tree_ring_memory_core::{ + models::{MemoryEvent, TreeRingResult}, + normalize_legacy_private_scope_identity, +}; use crate::sqlite_error_from_rusqlite; pub(crate) fn event_from_row(row: &Row<'_>) -> SqliteResult> { let raw_json: String = row.get(0)?; - Ok(serde_json::from_str::(&raw_json).map_err(Into::into)) + Ok((|| { + let mut event = serde_json::from_str::(&raw_json)?; + normalize_legacy_private_scope_identity(&mut event)?; + Ok(event) + })()) } pub(crate) fn collect_rows(rows: I) -> TreeRingResult> diff --git a/crates/tree-ring-memory-sqlite/src/write.rs b/crates/tree-ring-memory-sqlite/src/write.rs index 5771404..06f807f 100644 --- a/crates/tree-ring-memory-sqlite/src/write.rs +++ b/crates/tree-ring-memory-sqlite/src/write.rs @@ -1,4 +1,5 @@ use rusqlite::{params, OptionalExtension, Transaction}; +use sha2::{Digest, Sha256}; use std::time::Duration; use tree_ring_memory_core::models::{MemoryEvent, TreeRingError, TreeRingResult}; @@ -10,6 +11,40 @@ const WRITE_RETRY_ATTEMPTS: usize = 8; const WRITE_RETRY_INITIAL_DELAY_MS: u64 = 5; const WRITE_RETRY_MAX_DELAY_MS: u64 = 100; +pub(crate) const UPSERT_MEMORY_SQL: &str = r#" + INSERT INTO memories ( + id, created_at, updated_at, project, agent_profile, workflow_id, session_id, + operation_id, scope, ring, event_type, summary, details, source_json, + tags_json, salience, confidence, sensitivity, retention, expires_at, + supersedes_json, superseded_by, links_json, review_json, raw_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + created_at = excluded.created_at, + updated_at = excluded.updated_at, + project = excluded.project, + agent_profile = excluded.agent_profile, + workflow_id = excluded.workflow_id, + session_id = excluded.session_id, + operation_id = excluded.operation_id, + scope = excluded.scope, + ring = excluded.ring, + event_type = excluded.event_type, + summary = excluded.summary, + details = excluded.details, + source_json = excluded.source_json, + tags_json = excluded.tags_json, + salience = excluded.salience, + confidence = excluded.confidence, + sensitivity = excluded.sensitivity, + retention = excluded.retention, + expires_at = excluded.expires_at, + supersedes_json = excluded.supersedes_json, + superseded_by = excluded.superseded_by, + links_json = excluded.links_json, + review_json = excluded.review_json, + raw_json = excluded.raw_json + "#; + pub(crate) fn delete_in_transaction( transaction: &Transaction<'_>, memory_id: &str, @@ -20,6 +55,18 @@ pub(crate) fn delete_in_transaction( transaction .execute("DELETE FROM memory_fts WHERE id = ?", params![memory_id]) .map_err(sqlite_error_from_rusqlite)?; + transaction + .execute( + "DELETE FROM operation_claims WHERE memory_id = ?", + params![memory_id], + ) + .map_err(sqlite_error_from_rusqlite)?; + transaction + .execute( + "DELETE FROM redaction_tombstones WHERE memory_id = ?", + params![memory_id], + ) + .map_err(sqlite_error_from_rusqlite)?; Ok(deleted > 0) } @@ -39,27 +86,199 @@ pub(crate) fn redact_in_transaction( else { return Ok(false); }; + let superseded_by = event.superseded_by.clone(); event.redact(); + event.superseded_by = superseded_by; put_in_transaction(transaction, &event)?; + transaction + .execute( + r#" + INSERT OR IGNORE INTO redaction_tombstones (memory_id) + VALUES (?) + "#, + params![&event.id], + ) + .map_err(sqlite_error_from_rusqlite)?; Ok(true) } -pub(crate) fn put_in_transaction( +pub(crate) fn supersede_in_transaction( + transaction: &Transaction<'_>, + old_id: &str, + new_id: &str, +) -> TreeRingResult { + let Some(mut event) = transaction + .query_row( + "SELECT raw_json FROM memories WHERE id = ?", + params![old_id], + search::event_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose()? + else { + return Ok(false); + }; + event.superseded_by = Some(new_id.to_string()); + let raw_json = serde_json::to_string(&event)?; + transaction + .execute( + "UPDATE memories SET superseded_by = ?, raw_json = ? WHERE id = ?", + params![new_id, raw_json, old_id], + ) + .map_err(sqlite_error_from_rusqlite)?; + Ok(true) +} + +pub(crate) fn ensure_operation_claim_available( transaction: &Transaction<'_>, event: &MemoryEvent, ) -> TreeRingResult<()> { - let mut insert_memory = transaction - .prepare( + let Some(operation_id) = event.operation_id.as_deref() else { + return Ok(()); + }; + let claim_hash = operation_namespace_hash(event, operation_id); + let claimed: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM operation_claims WHERE namespace_hash = ?)", + params![claim_hash.as_slice()], + |row| row.get(0), + ) + .map_err(sqlite_error_from_rusqlite)?; + if claimed { + return Err(TreeRingError::Validation( + "operation_id was already used by a redacted memory".to_string(), + )); + } + Ok(()) +} + +pub(crate) fn prepare_memory_write( + transaction: &Transaction<'_>, + event: &MemoryEvent, +) -> TreeRingResult<()> { + event.validate()?; + let tombstoned: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM redaction_tombstones WHERE memory_id = ?)", + params![&event.id], + |row| row.get(0), + ) + .map_err(sqlite_error_from_rusqlite)?; + if tombstoned && !is_sanitized_redaction(event) { + return Err(TreeRingError::Validation( + "memory id was redacted and cannot be replaced; hard-delete it before reuse" + .to_string(), + )); + } + + let existing = transaction + .query_row( + "SELECT raw_json FROM memories WHERE id = ?", + params![&event.id], + search::event_from_row, + ) + .optional() + .map_err(sqlite_error_from_rusqlite)? + .transpose()?; + if let Some(existing) = existing { + preserve_replaced_operation_claim(transaction, &existing, event)?; + } + ensure_operation_claim_available(transaction, event) +} + +pub(crate) fn is_redaction_tombstoned( + transaction: &Transaction<'_>, + memory_id: &str, +) -> TreeRingResult { + transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM redaction_tombstones WHERE memory_id = ?)", + params![memory_id], + |row| row.get(0), + ) + .map_err(sqlite_error_from_rusqlite) +} + +fn preserve_replaced_operation_claim( + transaction: &Transaction<'_>, + existing: &MemoryEvent, + replacement: &MemoryEvent, +) -> TreeRingResult<()> { + let Some(existing_operation_id) = existing.operation_id.as_deref() else { + return Ok(()); + }; + let existing_hash = operation_namespace_hash(existing, existing_operation_id); + let replacement_hash = replacement + .operation_id + .as_deref() + .map(|operation_id| operation_namespace_hash(replacement, operation_id)); + if replacement_hash.as_ref() == Some(&existing_hash) { + return Ok(()); + } + transaction + .execute( r#" - INSERT OR REPLACE INTO memories ( - id, created_at, updated_at, project, agent_profile, scope, ring, - event_type, summary, details, source_json, tags_json, salience, - confidence, sensitivity, retention, expires_at, supersedes_json, - superseded_by, links_json, review_json, raw_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT OR IGNORE INTO operation_claims (namespace_hash, memory_id) + VALUES (?, ?) "#, + params![existing_hash.as_slice(), &existing.id], ) .map_err(sqlite_error_from_rusqlite)?; + Ok(()) +} + +fn is_sanitized_redaction(event: &MemoryEvent) -> bool { + event.summary == "[REDACTED]" + && event.details.is_empty() + && event.project.is_none() + && event.agent_profile.is_none() + && event.workflow_id.is_none() + && event.session_id.is_none() + && event.operation_id.is_none() + && event.event_type == "redacted" + && event.tags.is_empty() + && event.source.source_type == "manual" + && event.source.ref_.is_empty() + && event.source.quote.is_empty() + && event.supersedes.is_empty() + && event.links.is_empty() + && !event.review.needs_review + && event.review.review_reason.is_none() + && event.review.reviewed_at.is_none() + && event.review.reviewed_by.is_none() + && event.sensitivity == "private" +} + +pub(crate) fn operation_namespace_hash(event: &MemoryEvent, operation_id: &str) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"tree-ring-operation-namespace-v1"); + for value in [ + event.project.as_deref(), + event.workflow_id.as_deref(), + event.agent_profile.as_deref(), + Some(operation_id), + ] { + match value { + Some(value) => { + hasher.update([1]); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value.as_bytes()); + } + None => hasher.update([0]), + } + } + hasher.finalize().into() +} + +pub(crate) fn put_in_transaction( + transaction: &Transaction<'_>, + event: &MemoryEvent, +) -> TreeRingResult<()> { + prepare_memory_write(transaction, event)?; + let mut insert_memory = transaction + .prepare(UPSERT_MEMORY_SQL) + .map_err(sqlite_error_from_rusqlite)?; let mut delete_fts = transaction .prepare("DELETE FROM memory_fts WHERE id = ?") .map_err(sqlite_error_from_rusqlite)?; @@ -90,6 +309,9 @@ pub(crate) fn put_with_statements( &event.updated_at, event.project.as_deref(), event.agent_profile.as_deref(), + event.workflow_id.as_deref(), + event.session_id.as_deref(), + event.operation_id.as_deref(), &event.scope, &event.ring, &event.event_type, diff --git a/docs/architecture/rust-core-status.md b/docs/architecture/rust-core-status.md index 427645a..f4918c2 100644 --- a/docs/architecture/rust-core-status.md +++ b/docs/architecture/rust-core-status.md @@ -6,18 +6,32 @@ the v0.4 Rust-owned JSONL import/export path, v0.5 deterministic audit checks, v0.6 deterministic consolidation, v0.7 Rust-owned maintenance, v0.8 Python-runtime removal, v0.9 removal of tracked Python source and optional CPython bindings from the canonical repo, v0.10 installer/onboarding work, and -v0.11 Rust-native source adapters plus framework discovery. +v0.11 Rust-native source adapters plus framework discovery. The v0.12 line now +also carries explicit same-host multi-agent correlation, partitioning, and +idempotent-write semantics. ## Current Status - The public runtime is Rust-native through the Rust CLI and Rust crates. - Rust workspace exists under `crates/`. - Rust core owns model validation, sensitivity checks, and recall scoring. +- The portable event model includes optional `workflow_id`, `session_id`, and + `operation_id` alongside `agent_profile`. Agent, workflow, and session scopes + require their matching partition identifier. Scope is routing metadata, not + an authorization boundary. - Rust SQLite crate owns schema-compatible SQLite/FTS storage. +- SQLite migration and read boundaries normalize identity-less pre-0.12 private + scopes into deterministic per-record legacy partitions marked for review. + Redaction retains a memory-ID tombstone, and replaced operation namespaces + remain claimed until explicit hard deletion. - Rust CLI owns the full local command surface: init, remember, evidence, recall, forget, import/export, audit, consolidate, maintain, DOX sync, Revolve sync, framework discovery, welcome onboarding, and TUI operation. - Rust CLI has JSON output for machine-readable adapter use. +- Remember, evidence, recall, and consolidation expose agent/workflow/session + context through CLI flags; agent profile, workflow ID, and session ID also + have `TREE_RING_*` environment defaults. Exact operation retries are stable, + while conflicting operation-key reuse fails closed. - CLI and TUI durable operations now share action request/report contracts for behavior-preserving command execution. This keeps CLI output ownership, TUI state/render ownership, and storage ownership separate while preparing the @@ -85,6 +99,27 @@ v0.11 Rust-native source adapters plus framework discovery. leaves global harness configuration opt-in, and keeps durable memory updates agent-mediated instead of background-recorded. +## Multi-Agent Evidence Boundary + +`crates/tree-ring-memory-cli/tests/multi_agent_acceptance.rs` exercises the +public binary as a coordinator would: + +- A parent connection holds `BEGIN IMMEDIATE` while eight real `tree-ring` + worker processes start, and the test verifies every worker is waiting before + releasing the gate. +- Every worker writes agent-scoped JSON with a unique profile/operation and a + shared workflow/session. +- Recall assertions independently exercise profile, workflow, session, and + scope filters, plus the intended fan-in across profiles. +- An exact operation retry returns the original memory ID, conflicting reuse + exits nonzero, and the JSON maintenance report proves exact memory-row/FTS + parity with no missing or orphan rows. + +This is bounded evidence for concurrent processes sharing a local SQLite store +on one host. It is not evidence for sustained load, fairness, abrupt-process +crash recovery, cross-host coordination, NFS/network-filesystem safety, or a +distributed lock service. + ## Build Commands ```bash @@ -111,12 +146,14 @@ sh scripts/certify-tree-ring.sh SQLite/FTS storage, transactional row/FTS consistency, redaction, JSONL import/export filtering and duplicate handling, deterministic audit checks, deterministic consolidation planning, maintenance planning/application, FTS - repair, and basic concurrent writes. + repair, deterministic operation idempotency, and concurrent writes. - Rust CLI tests cover the scriptable init/remember/recall/forget commands and JSONL import/export/audit/consolidate commands plus the Ratatui TUI model, stream reader, slash-command parser, store-watch refresh, confirmation-gated actions, DOX/Revolve sync commands, framework discovery, CLI parsing, and - render-buffer smoke. + render-buffer smoke. The process-level multi-agent acceptance test adds + deterministic write-lock contention, routing-filter isolation, idempotency + conflict handling, and row/FTS parity through the public CLI. - `crates/tree-ring-memory-sqlite/examples/performance_smoke.rs` provides an operator-run local insert and recall timing check. It fails if expected recalls are empty, emits a stable `METRICS_JSON=` line, and enforces diff --git a/docs/integrations/agent-skill.md b/docs/integrations/agent-skill.md index 7598338..97cd18f 100644 --- a/docs/integrations/agent-skill.md +++ b/docs/integrations/agent-skill.md @@ -68,6 +68,129 @@ For a project-local install, use the generated quick reference in .tree-ring/bin/tree-ring --root .tree-ring tui ``` +## Multi-Agent Fan-Out And Fan-In + +Tree Ring can coordinate multiple CLI workers that share one local memory root +on one host. The coordinator chooses one project, workflow ID, and session ID. +Every worker gets a unique agent profile and a stable operation ID for each +logical write: + +```bash +tree-ring --root .tree-ring init + +tree-ring --root .tree-ring remember "Storage worker validated WAL behavior." \ + --event-type lesson \ + --scope agent \ + --project example-service \ + --agent-profile worker-storage \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --operation-id validate-storage-v1 \ + --source-ref runs/release-readiness/worker-storage.json \ + --tag coordination +``` + +Repeat the worker command with a different `--agent-profile`, +`--operation-id`, and source reference. Keep `--workflow-id` and +`--session-id` shared across that fan-out. An exact retry of one logical write +reuses its original session and operation IDs. Rotate the session and assign new +operation IDs only when starting a genuinely new execution attempt. + +Scope determines the required partition identity: + +| Scope | Required identity | Intended use | +| --- | --- | --- | +| `agent` | `agent_profile` | One worker's partitioned task memory | +| `workflow` | `workflow_id` | Shared state for one fan-out/fan-in | +| `session` | `session_id` | State partitioned by execution attempt | +| `project` | None; `project` is recommended | Shared repository memory | +| `global` | None | Deliberate cross-project memory | + +The CLI also reads `TREE_RING_AGENT_PROFILE`, `TREE_RING_WORKFLOW_ID`, and +`TREE_RING_SESSION_ID`. Explicit flags are easier to audit in retained worker +commands. If the coordinator uses environment defaults, clear the agent-profile +default before recalling all workers; otherwise it becomes an unintended recall +filter. + +At fan-in, recall worker results through the shared dimensions: + +```bash +tree-ring --root .tree-ring --json recall "release readiness" \ + --project example-service \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --scope agent \ + --limit 64 +``` + +Inspect each memory's source reference before writing a coordinator-owned +summary. Use `scope=workflow` with the same workflow ID for shared workflow +state, or `scope=project` for a reviewed durable project conclusion: + +```bash +tree-ring --root .tree-ring remember "Release readiness checks passed." \ + --event-type summary \ + --scope workflow \ + --project example-service \ + --agent-profile coordinator \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --operation-id fan-in-summary-v1 \ + --source-ref runs/release-readiness/fan-in.json +``` + +Consolidation accepts `--agent-profile`, `--workflow-id`, and `--session-id` +filters. Agent-, workflow-, and session-scoped inputs remain partitioned; +consolidation does not silently merge memories from different partition +identities. A coordinator that wants a shared conclusion should write the +explicit source-linked summary shown above. + +### Idempotency + +`operation_id` is an idempotency key inside the +`(project, workflow_id, agent_profile)` namespace: + +- An exact retry with the same payload returns the existing memory ID and does + not add a row. +- Reusing the key in that namespace for a different payload fails nonzero. +- `session_id` is retained context but is not part of the idempotency namespace; + changing only the session makes the retry conflict. +- Omitting `operation_id` creates an ordinary new write. +- Consolidation-derived summaries do not inherit a source operation ID. +- Replacing an active memory ID preserves its old operation namespace as a + one-way claim. +- Redaction keeps both the one-way operation claim and a memory-ID tombstone, + so retries and replacement imports fail closed without restoring redacted + content. Only hard deletion removes those claims. + +Pre-0.12 `agent`, `workflow`, or `session` records that lack the now-required +identity are assigned a deterministic, per-record `legacy-*` identity during +migration/import and marked for review. They remain privately partitioned and +portable instead of being widened into project/global scope. + +The identifiers must be nonblank, contain no control characters, and stay at or +below 256 characters. These fields are routing and correlation metadata, not an +authorization boundary. A process with filesystem access to the SQLite store +can read it, so host permissions still control access. + +### Runtime Boundary And Evidence + +The supported shared-root pattern is concurrent processes on the same host +using a local filesystem. SQLite WAL, bounded lock retries, and a busy timeout +handle local contention. Tree Ring is not a distributed database or lock +service, and this evidence does not establish safe shared-database operation +over NFS, network filesystems, containers on different hosts, or multiple +machines. Use per-host roots and an explicit export/import or other +evidence-preserving coordinator when work spans hosts. + +`crates/tree-ring-memory-cli/tests/multi_agent_acceptance.rs` is a bounded +process-level acceptance test. It launches eight real CLI writers against one +root, exercises profile/workflow/session/scope recall filters, verifies exact +retry and conflicting-key behavior, and checks memory-row/FTS parity through +the JSON maintenance report. It is evidence for the same-host contract only; +it is not a sustained-load, crash-recovery, fairness, or distributed-storage +certification. + ## Evidence-Driven Improvement Use `tree-ring evidence` when a lesson comes from an evaluation, checkpoint, diff --git a/docs/protocol/memory-event.md b/docs/protocol/memory-event.md index 95cf837..98da38c 100644 --- a/docs/protocol/memory-event.md +++ b/docs/protocol/memory-event.md @@ -2,7 +2,78 @@ `MemoryEvent` is the portable unit of Tree Ring Memory. -The event is not a transcript line. It is a meaningful memory statement with scope, ring, source evidence, confidence, salience, sensitivity, retention, and review state. +The event is not a transcript line. It is a meaningful memory statement with +scope, ring, source evidence, confidence, salience, sensitivity, retention, +review state, and optional multi-agent correlation metadata. + +## Multi-Agent Context + +The portable JSON event supports these optional fields: + +- `project`: project or repository label +- `agent_profile`: producer role or worker identity +- `workflow_id`: shared fan-out/fan-in correlation ID +- `session_id`: one execution-attempt correlation ID +- `operation_id`: idempotency key for one logical write + +Missing fields deserialize as `null`. At import, read, and SQLite migration +boundaries, a pre-0.12 `agent`, `workflow`, or `session` record missing its +required identity receives a deterministic, non-sensitive, per-record +`legacy-*` identity and is marked for review. New events remain strict and +cannot be written identity-less. When present, each value must be nonblank, +contain no control characters, and contain at most 256 Unicode characters. +Sensitivity inspection includes all five fields. + +Scope establishes a partition invariant: + +| Scope | Required field | Meaning | +| --- | --- | --- | +| `agent` | `agent_profile` | Partition by producer identity | +| `workflow` | `workflow_id` | Partition by coordinated workflow | +| `session` | `session_id` | Partition by execution attempt | +| `project` | None | Shared project memory; project-local roots may omit `project` | +| `global` | None | Deliberate cross-project memory | + +Other supported scopes retain their existing meanings. Scope and identity are +routing metadata, not an ACL or authentication boundary. Any process with +filesystem access to the local store can issue unfiltered recall. + +## Idempotent Writes + +When `operation_id` is present, SQLite resolves it inside the +`(project, workflow_id, agent_profile)` namespace: + +- An exact replay of the same write returns the existing memory ID. +- A different payload using the same namespaced key fails closed. +- `session_id` is retained in the payload but is not part of that namespace, so + changing only the session is a conflicting reuse. +- Without `operation_id`, each accepted command is an ordinary new write. +- Replacing an active row under the same memory ID preserves its prior + operation namespace as a one-way claim. + +Derived consolidation memories always use `operation_id: null`; they must not +reuse a source event's write key. Redaction clears all identity/correlation +metadata. If the original scope was `agent`, `workflow`, or `session`, +redaction changes it to `manual` so the redacted event remains valid without +retaining its partition identifier. Storage retains only a length-delimited +SHA-256 namespace claim plus the memory ID, and a separate memory-ID tombstone, +so neither an automated retry nor replacement import can recreate redacted +content while the raw project, profile, workflow, session, and operation values +remain scrubbed. Only explicit hard deletion removes those claims. + +## Partition-Aware Lifecycle Behavior + +Consolidation keeps agent-scoped groups separate by `agent_profile`, +workflow-scoped groups separate by `workflow_id`, and session-scoped groups +separate by `session_id`. Output summaries retain producer/correlation fields +only when every contributing event agrees; a shared project/global summary does +not claim one producer when sources differ. Source-memory links remain the +provenance trail. + +Contradiction audit uses the same scoped partitions, preventing +cross-worker, cross-workflow, or cross-session false positives. Project and +global directives remain shared and can surface contradictions across +producers. ## Rings @@ -15,8 +86,21 @@ The event is not a transcript line. It is a meaningful memory statement with sco ## Recall Defaults -Recall excludes sensitive and superseded memory unless explicitly requested. Results should include source evidence and ranking explanation when `explain_ranking` is true. +Recall excludes sensitive and superseded memory unless explicitly requested. +Callers may filter by project, agent profile, workflow ID, session ID, and +scope. A coordinator intentionally omits the agent-profile filter when +collecting all worker results in one workflow/session. Results should include +source evidence and ranking explanation when `explain_ranking` is true. ## Privacy Defaults -Secrets are blocked by default. Sensitive memory is excluded from recall and export by default. +Secrets are blocked by default. Sensitive memory is excluded from recall and +export by default. + +## Storage Boundary + +The shared-root contract covers concurrent processes on one host using a local +filesystem. SQLite WAL, a busy timeout, and bounded lock retries handle this +local contention. The protocol does not claim distributed locking, multi-host +database coordination, or safe SQLite sharing over NFS or other network +filesystems. diff --git a/fixtures/parity/legacy-private-scope-blank-identity.jsonl b/fixtures/parity/legacy-private-scope-blank-identity.jsonl new file mode 100644 index 0000000..8334a6b --- /dev/null +++ b/fixtures/parity/legacy-private-scope-blank-identity.jsonl @@ -0,0 +1 @@ +{"id":"mem_legacy_blank_agent_001","created_at":"2025-11-01T12:04:00Z","updated_at":"2025-11-01T12:04:00Z","agent_profile":" \t ","scope":"agent","event_type":"lesson","summary":"Legacy agent-scoped memory with a blank identity."} diff --git a/fixtures/parity/legacy-private-scopes-export.jsonl b/fixtures/parity/legacy-private-scopes-export.jsonl new file mode 100644 index 0000000..dc1f410 --- /dev/null +++ b/fixtures/parity/legacy-private-scopes-export.jsonl @@ -0,0 +1,4 @@ +{"type":"tree_ring_memory_export","schema_version":1,"plugin_version":"0.11.0","created_at":"2025-11-01T12:03:00Z","memory_count":3,"sensitive_included":false} +{"type":"memory_event","memory":{"id":"mem_legacy_agent_001","created_at":"2025-11-01T12:00:00Z","updated_at":"2025-11-01T12:00:00Z","scope":"agent","event_type":"lesson","summary":"Legacy agent-scoped memory."}} +{"type":"memory_event","memory":{"id":"mem_legacy_workflow_001","created_at":"2025-11-01T12:01:00Z","updated_at":"2025-11-01T12:01:00Z","scope":"workflow","event_type":"decision","summary":"Legacy workflow-scoped memory."}} +{"type":"memory_event","memory":{"id":"mem_legacy_session_001","created_at":"2025-11-01T12:02:00Z","updated_at":"2025-11-01T12:02:00Z","scope":"session","event_type":"observation","summary":"Legacy session-scoped memory."}} diff --git a/fixtures/parity/legacy-private-scopes-raw.jsonl b/fixtures/parity/legacy-private-scopes-raw.jsonl new file mode 100644 index 0000000..71d4b0d --- /dev/null +++ b/fixtures/parity/legacy-private-scopes-raw.jsonl @@ -0,0 +1,3 @@ +{"id":"mem_legacy_agent_001","created_at":"2025-11-01T12:00:00Z","updated_at":"2025-11-01T12:00:00Z","scope":"agent","event_type":"lesson","summary":"Legacy agent-scoped memory."} +{"id":"mem_legacy_workflow_001","created_at":"2025-11-01T12:01:00Z","updated_at":"2025-11-01T12:01:00Z","scope":"workflow","event_type":"decision","summary":"Legacy workflow-scoped memory."} +{"id":"mem_legacy_session_001","created_at":"2025-11-01T12:02:00Z","updated_at":"2025-11-01T12:02:00Z","scope":"session","event_type":"observation","summary":"Legacy session-scoped memory."} diff --git a/schemas/memory-event.schema.json b/schemas/memory-event.schema.json index c2fb717..9d701c3 100644 --- a/schemas/memory-event.schema.json +++ b/schemas/memory-event.schema.json @@ -8,9 +8,12 @@ "id": { "type": "string", "minLength": 1 }, "created_at": { "type": "string", "format": "date-time" }, "updated_at": { "type": "string", "format": "date-time" }, - "project": { "type": ["string", "null"] }, - "agent_profile": { "type": ["string", "null"] }, - "scope": { "enum": ["global", "project", "agent", "session", "workflow", "tool", "eval", "manual"] }, + "project": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "agent_profile": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "workflow_id": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "session_id": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "operation_id": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "scope": { "enum": ["global", "project", "agent", "session", "workflow", "tool", "eval", "manual", "dox", "revolve"] }, "ring": { "enum": ["cambium", "outer", "inner", "heartwood", "scar", "seed"] }, "event_type": { "type": "string", "minLength": 1 }, "summary": { "type": "string", "minLength": 1 }, @@ -57,5 +60,19 @@ "additionalProperties": false } }, + "allOf": [ + { + "if": { "properties": { "scope": { "const": "agent" } }, "required": ["scope"] }, + "then": { "properties": { "agent_profile": { "type": "string", "minLength": 1 } }, "required": ["agent_profile"] } + }, + { + "if": { "properties": { "scope": { "const": "workflow" } }, "required": ["scope"] }, + "then": { "properties": { "workflow_id": { "type": "string", "minLength": 1 } }, "required": ["workflow_id"] } + }, + { + "if": { "properties": { "scope": { "const": "session" } }, "required": ["scope"] }, + "then": { "properties": { "session_id": { "type": "string", "minLength": 1 } }, "required": ["session_id"] } + } + ], "additionalProperties": false } diff --git a/schemas/recall-query.schema.json b/schemas/recall-query.schema.json index 84ccb75..e76ed51 100644 --- a/schemas/recall-query.schema.json +++ b/schemas/recall-query.schema.json @@ -6,9 +6,13 @@ "required": ["query"], "properties": { "query": { "type": "string" }, - "project": { "type": ["string", "null"] }, - "agent_profile": { "type": ["string", "null"] }, - "scope": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "agent_profile": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "workflow_id": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "session_id": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "scope": { + "enum": ["global", "project", "agent", "session", "workflow", "tool", "eval", "manual", "dox", "revolve", null] + }, "rings": { "type": "array", "items": { "type": "string" } }, "event_types": { "type": "array", "items": { "type": "string" } }, "include_sensitive": { "type": "boolean", "default": false }, diff --git a/scripts/certify-tree-ring.sh b/scripts/certify-tree-ring.sh index 9bd3de2..328257b 100755 --- a/scripts/certify-tree-ring.sh +++ b/scripts/certify-tree-ring.sh @@ -3,16 +3,29 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) OUT_DIR="$ROOT/target/tree-ring-certification" -TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/tree-ring-cert.XXXXXX") +mkdir -p "$OUT_DIR" +RUN_LOCK="$OUT_DIR/.certification-run.lock" +RUN_LOCK_OWNED=0 +TMP_DIR="" +METRICS_TMP="$OUT_DIR/.metrics.$$.tmp" +SUMMARY_TMP="$OUT_DIR/.summary.$$.tmp" BIN="$ROOT/target/release/tree-ring" IMPORT_COUNT=${TREE_RING_CERT_IMPORT_COUNT:-10000} EXTENDED=${TREE_RING_CERT_EXTENDED:-0} AGENT_ZERO_ROOT=${TREE_RING_AGENT_ZERO_ROOT:-} cleanup() { - rm -rf "$TMP_DIR" + [ -z "$TMP_DIR" ] || rm -rf "$TMP_DIR" + rm -f "$METRICS_TMP" "$SUMMARY_TMP" + [ "$RUN_LOCK_OWNED" -eq 0 ] || rmdir "$RUN_LOCK" 2>/dev/null || true } trap cleanup EXIT INT TERM +if ! mkdir "$RUN_LOCK" 2>/dev/null; then + printf 'Tree Ring certification failed: another certification run owns %s\n' "$RUN_LOCK" >&2 + exit 1 +fi +RUN_LOCK_OWNED=1 +TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/tree-ring-cert.XXXXXX") log() { printf '==> %s\n' "$*" @@ -121,14 +134,13 @@ if errors: PY } -mkdir -p "$OUT_DIR" SUMMARY="$OUT_DIR/summary.md" METRICS="$OUT_DIR/metrics.json" INDEX="$OUT_DIR/evidence-index.json" LOG="$OUT_DIR/certification.log" QUALITY_OUT="$OUT_DIR/quality" QUALITY_RUN_OUT="$OUT_DIR/quality-run.out" -rm -f "$SUMMARY" "$METRICS" "$QUALITY_RUN_OUT" +rm -f "$QUALITY_RUN_OUT" rm -rf "$QUALITY_OUT" : > "$LOG" @@ -295,7 +307,7 @@ if [ -n "$AGENT_ZERO_ROOT" ]; then fi created_at=$(now_utc) -cat > "$METRICS" < "$METRICS_TMP" < "$METRICS" < "$SUMMARY" < "$SUMMARY_TMP" < "$INDEX" < "$OUT_DIR/harness-certification.json" diff --git a/skills/tree-ring-memory/SKILL.md b/skills/tree-ring-memory/SKILL.md index 8c746ec..b93f7af 100644 --- a/skills/tree-ring-memory/SKILL.md +++ b/skills/tree-ring-memory/SKILL.md @@ -1,7 +1,7 @@ --- name: tree-ring-memory description: Guides AI agents in using Tree Ring Memory for durable recall, project decisions, user preferences, warnings, future seeds, privacy-safe memory capture, and lifecycle-aware forgetting. -version: 0.11.0 +version: 0.12.0 tags: ["memory", "agents", "recall", "privacy", "projects", "dox", "revolve", "skills", "cli"] triggers: - "remember this" @@ -14,6 +14,7 @@ triggers: - "sync DOX" - "sync Revolve" - "evidence loop" + - "multi-agent memory" --- # Tree Ring Memory @@ -175,7 +176,9 @@ If a useful memory contains sensitive material, store a redacted summary with en Set project and scope deliberately: - use project scope for repo-specific rules, decisions, warnings, and lessons -- use agent scope for agent-profile behavior +- use agent scope for agent-partitioned behavior and always set `agent_profile` +- use workflow scope for one coordinated fan-out/fan-in and always set `workflow_id` +- use session scope for one execution attempt and always set `session_id` - use global scope only for durable user preferences or cross-project guidance - include source references such as file paths, issue ids, PR ids, run ids, or docs paths - use `tree-ring evidence ... --evidence-ref ` for evaluated outcomes @@ -189,6 +192,57 @@ When DOX or Revolve source records change, re-run the matching sync adapter with `--dry-run`, inspect the generated memories, then run the write command only when the summaries are useful and source-linked. +## Multi-Agent Coordination + +For workers sharing one local Tree Ring root, give every write explicit +coordination metadata: + +```bash +tree-ring --root .tree-ring remember "Worker validated the storage boundary." \ + --event-type lesson \ + --scope agent \ + --project example-service \ + --agent-profile worker-storage \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --operation-id validate-storage-v1 \ + --source-ref runs/release-readiness/worker-storage.json +``` + +Use a unique `agent_profile` per worker, one shared `workflow_id` for the +fan-out/fan-in, one `session_id` for each genuine execution attempt, and a stable +unique `operation_id` for each logical write. An exact retry reuses both the +original session ID and operation ID; changing only the session is a conflicting +reuse. Start a new session and use new operation IDs only for a genuinely new +attempt. Exact retries with the same operation metadata and payload return the +original memory. Reusing that operation key for a different payload fails +closed. Replacing a stored memory keeps its old operation namespace claimed. +Redaction also tombstones the memory ID; only an explicit hard delete releases +those claims. + +At fan-in, recall the shared workflow and session without an agent-profile +filter, inspect the source refs, then write a source-linked workflow or project +summary: + +```bash +tree-ring --root .tree-ring recall "release readiness" \ + --project example-service \ + --workflow-id release-readiness \ + --session-id attempt-1 \ + --scope agent +``` + +`TREE_RING_AGENT_PROFILE`, `TREE_RING_WORKFLOW_ID`, and +`TREE_RING_SESSION_ID` provide the same defaults as their CLI flags. Do not +leave an agent-profile environment filter set when the coordinator intends to +recall every worker. + +This shared-root pattern is for concurrent processes on one host using a local +filesystem. It is not a distributed lock service and does not claim safe +cross-host or NFS operation. Scope and identity fields are routing metadata, not +an ACL; a same-user coordinator can recall across profiles. Use per-host stores +plus an explicit, evidence-preserving fan-in process when work spans hosts. + ## Agent-Mediated Updates Tree Ring Memory does not autonomously scrape chats or write durable memory in @@ -214,6 +268,10 @@ If memory is wrong, private, stale, or superseded: - supersede it when a newer decision replaces it - prefer explicit reasons for every forget operation +Treat redaction as monotonic. Do not try to restore a redacted ID through +replacement import; create a new reviewed memory only if the user deliberately +reintroduces safe content. + Never keep known-wrong memory merely because it was previously recalled. ## Closeout Habit diff --git a/templates/dox/AGENTS.md b/templates/dox/AGENTS.md index bd5807c..9f36449 100644 --- a/templates/dox/AGENTS.md +++ b/templates/dox/AGENTS.md @@ -74,6 +74,36 @@ the CLI. It must not be used as a hidden transcript recorder. Bridge files tell the active agent when to call Tree Ring; they do not authorize autonomous background capture. +## Multi-Agent Coordination + +When multiple local workers share this memory root: + +- Give every worker a unique `agent_profile`. +- Share one `workflow_id` across the fan-out/fan-in. +- Use one `session_id` for each genuine execution attempt; exact retries reuse + the original session ID. +- Give each logical write a stable unique `operation_id` and a source reference. +- Use `scope=agent` only with `agent_profile`, `scope=workflow` only with + `workflow_id`, and `scope=session` only with `session_id`. +- Recall at fan-in with explicit workflow, session, and scope filters. Omit the + agent-profile filter when the coordinator needs every worker. +- Treat project and global memories as shared. Do not attribute a shared summary + to one worker unless every source has that producer identity. + +`TREE_RING_AGENT_PROFILE`, `TREE_RING_WORKFLOW_ID`, and +`TREE_RING_SESSION_ID` can provide the matching CLI defaults. An exact retry of +the same operation and payload returns the original memory; conflicting reuse +of the operation key must fail. Replaced operation namespaces and redacted +memory IDs stay claimed until explicit hard deletion. + +Scope and identity fields are routing partitions, not authorization boundaries. +A same-user coordinator with filesystem access can recall across worker +profiles. + +This shared-root contract covers concurrent processes on one host and a local +filesystem. It is not a distributed lock service and does not claim safe +cross-host or NFS database sharing. + ## Ring Mapping - Use `cambium` for active task context.