Add coding-agent memory dataset seed

data/codingagent/synthetic/cases.json

@@ -0,0 +1,98 @@
+[
+  {
+    "id": "worker-id-stability",
+    "category": "ci_failure_recurrence",
+    "documents": [
+      {
+        "id": "session-1",
+        "timestamp": "2026-06-20T09:30:00Z",
+        "content": "Session 1: A container deployment used the hostname as the default worker id. The team saw pending consolidation operations that were not claimed after a restart. The accepted fix direction was to require a stable HINDSIGHT_API_WORKER_ID in container deployments and warn when the process falls back to a container hostname."
+      },
+      {
+        "id": "session-2",
+        "timestamp": "2026-06-21T11:10:00Z",
+        "content": "Session 2: A follow-up noted that the warning must cover both API-embedded workers and standalone worker processes. Explicit --worker-id and HINDSIGHT_API_WORKER_ID should stay quiet; only the default hostname fallback should warn."
+      },
+      {
+        "id": "session-3",
+        "timestamp": "2026-06-22T15:40:00Z",
+        "content": "Session 3: Regression tests should simulate a container hostname and verify three paths: warning on fallback, no warning for an explicit worker id, and no warning outside a container."
+      }
+    ],
+    "queries": [
+      {
+        "id": "q1",
+        "category": "ci_failure_recurrence",
+        "query": "A new standalone worker test fails because it expects no warning when --worker-id is provided. What prior decision should guide the fix?",
+        "gold_document_ids": ["session-2", "session-3"],
+        "gold_answers": [
+          "Explicit worker ids should not warn; only the default hostname fallback inside a container should warn, and the regression tests should cover explicit id, fallback warning, and non-container paths."
+        ]
+      }
+    ]
+  },
+  {
+    "id": "retrieval-limit-wiring",
+    "category": "review_preference",
+    "documents": [
+      {
+        "id": "session-1",
+        "timestamp": "2026-06-18T08:00:00Z",
+        "content": "Session 1: The benchmark runner uses provider retrieval results to build a RAG context. The review preference is to keep benchmark fixes narrow and avoid changing scoring, prompts, or provider APIs unless a bug requires it."
+      },
+      {
+        "id": "session-2",
+        "timestamp": "2026-06-19T13:30:00Z",
+        "content": "Session 2: A bug was found in agentic-rag mode: the configured retrieval limit was parsed but not passed into the memory provider, so a mode using k=3 could still retrieve the provider default. The expected fix is to thread the limit through the existing call rather than add a new retrieval abstraction."
+      },
+      {
+        "id": "session-3",
+        "timestamp": "2026-06-21T10:15:00Z",
+        "content": "Session 3: A maintainer asked for small benchmark PRs that isolate one behavior and include a local command that proves the path still runs."
+      }
+    ],
+    "queries": [
+      {
+        "id": "q1",
+        "category": "review_preference",
+        "query": "When fixing another agentic-rag retrieval-count bug, what implementation style should be preferred?",
+        "gold_document_ids": ["session-1", "session-2", "session-3"],
+        "gold_answers": [
+          "Keep the patch narrow: pass the existing retrieval limit through the current provider call, avoid unrelated scoring or prompt changes, and include a local command proving the path still runs."
+        ]
+      }
+    ]
+  },
+  {
+    "id": "stale-docs-boundary",
+    "category": "release_or_docs_sync",
+    "documents": [
+      {
+        "id": "session-1",
+        "timestamp": "2026-05-07T10:00:00Z",
+        "content": "Session 1: An old Windows setup note said Hindsight v0.6.0 was current and recommended checking /metrics to verify that the API was running."
+      },
+      {
+        "id": "session-2",
+        "timestamp": "2026-06-18T09:36:01Z",
+        "content": "Session 2: Release v0.8.3 became the current Hindsight release. Setup notes that mention v0.6.0 as current should be treated as stale unless they are explicitly historical."
+      },
+      {
+        "id": "session-3",
+        "timestamp": "2026-06-20T08:44:00Z",
+        "content": "Session 3: For Chinese-network Windows installs, users reported that HF_ENDPOINT can help local embedding downloads, but FlashRank may need a direct proxy or manual cache download if the mirror does not contain the model."
+      }
+    ],
+    "queries": [
+      {
+        "id": "q1",
+        "category": "release_or_docs_sync",
+        "query": "A docs reply asks whether v0.6.0 is still the current recommended version for Windows setup. What should the answer do with the old memory?",
+        "gold_document_ids": ["session-1", "session-2"],
+        "gold_answers": [
+          "It should treat the v0.6.0 note as stale because v0.8.3 is current, while preserving any still-useful setup troubleshooting as historical context."
+        ]
+      }
+    ]
+  }
+]

src/memory_bench/dataset/__init__.py

@@ -1,5 +1,6 @@
 from .base import Dataset
 from .beam import BEAMDataset
+from .codingagent import CodingAgentDataset
 from .lifebench import LifeBenchDataset
 from .locomo import LoComoDataset
 from .longmemeval import LongMemEvalDataset
@@ -9,6 +10,7 @@
 
 REGISTRY: dict[str, type[Dataset]] = {
     "beam":         BEAMDataset,
+    "codingagent":  CodingAgentDataset,
     "lifebench":    LifeBenchDataset,
     "locomo":       LoComoDataset,
     "longmemeval":  LongMemEvalDataset,

src/memory_bench/dataset/codingagent.py

@@ -0,0 +1,138 @@
+"""
+Synthetic coding-agent memory dataset.
+
+This small seed split tests whether a memory system helps with engineering
+continuity across sessions: reusing prior decisions, avoiding stale facts, and
+remembering CI/review constraints without loading a whole repo into context.
+"""
+import json
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from .base import Dataset
+from ..models import Document, Query
+
+SPLITS = ["synthetic"]
+
+_CATEGORIES = [
+    "ci_failure_recurrence",
+    "review_preference",
+    "release_or_docs_sync",
+]
+
+
+class CodingAgentDataset(Dataset):
+    """Small synthetic dataset for coding-agent memory behavior."""
+
+    name = "codingagent"
+    description = "Synthetic multi-session engineering traces for coding-agent memory."
+    splits = SPLITS
+    task_type = "open"
+    isolation_unit = "case"
+    links = [
+        {"label": "Hindsight issue", "url": "https://github.com/vectorize-io/hindsight/issues/2347"},
+    ]
+
+    def _data_path(self, split: str) -> Path:
+        if split not in SPLITS:
+            raise ValueError(f"Unknown CodingAgent split '{split}'. Available: {SPLITS}")
+        return Path(__file__).parents[3] / "data" / "codingagent" / split / "cases.json"
+
+    def _load_cases(self, split: str) -> list[dict]:
+        with open(self._data_path(split), encoding="utf-8") as f:
+            return json.load(f)
+
+    def categories(self, split: str) -> list[str] | None:
+        return _CATEGORIES
+
+    def category_type(self, split: str, category: str) -> str:
+        return "query"
+
+    def get_result_categories(self, meta: dict) -> dict[str, list[str]]:
+        axes: dict[str, list[str]] = {}
+        if category := meta.get("category"):
+            axes["Category"] = [category]
+        if case_id := meta.get("case_id"):
+            axes["Case"] = [case_id]
+        return axes
+
+    def load_queries(
+        self,
+        split: str,
+        category: str | None = None,
+        limit: int | None = None,
+    ) -> list[Query]:
+        queries: list[Query] = []
+        for case in self._load_cases(split):
+            case_id = case["id"]
+            for raw_query in case.get("queries", []):
+                query_category = raw_query.get("category", case.get("category"))
+                if category and query_category != category:
+                    continue
+                queries.append(Query(
+                    id=f"{case_id}_{raw_query['id']}",
+                    query=raw_query["query"],
+                    gold_ids=[
+                        f"{case_id}_{doc_id}"
+                        for doc_id in raw_query.get("gold_document_ids", [])
+                    ],
+                    gold_answers=raw_query.get("gold_answers", []),
+                    user_id=case_id,
+                    meta={"case_id": case_id, "category": query_category},
+                ))
+
+        if limit:
+            queries = queries[:limit]
+        return queries
+
+    def load_documents(
+        self,
+        split: str,
+        category: str | None = None,
+        limit: int | None = None,
+        ids: set[str] | None = None,
+        user_ids: set[str] | None = None,
+    ) -> list[Document]:
+        documents: list[Document] = []
+        for case in self._load_cases(split):
+            case_id = case["id"]
+            if user_ids is not None and case_id not in user_ids:
+                continue
+            for raw_doc in case.get("documents", []):
+                doc_id = f"{case_id}_{raw_doc['id']}"
+                if ids is not None and doc_id not in ids:
+                    continue
+                documents.append(Document(
+                    id=doc_id,
+                    content=raw_doc["content"],
+                    user_id=case_id,
+                    timestamp=raw_doc.get("timestamp"),
+                    context=f"codingagent:{case_id}",
+                ))
+
+        if limit and ids is None:
+            documents = documents[:limit]
+        return documents
+
+    def dataset_stats(self, console: Console, **_) -> None:
+        table = Table(title="CodingAgent synthetic dataset stats")
+        table.add_column("Split", style="bold")
+        table.add_column("Cases", justify="right")
+        table.add_column("Documents", justify="right")
+        table.add_column("Queries", justify="right")
+        table.add_column("Categories", justify="right")
+
+        for split in SPLITS:
+            cases = self._load_cases(split)
+            documents = sum(len(case.get("documents", [])) for case in cases)
+            queries = sum(len(case.get("queries", [])) for case in cases)
+            categories = {
+                query.get("category", case.get("category"))
+                for case in cases
+                for query in case.get("queries", [])
+            }
+            table.add_row(split, str(len(cases)), str(documents), str(queries), str(len(categories)))
+
+        console.print(table)