diff --git a/data/codingagent/synthetic/cases.json b/data/codingagent/synthetic/cases.json new file mode 100644 index 0000000..d4c9938 --- /dev/null +++ b/data/codingagent/synthetic/cases.json @@ -0,0 +1,98 @@ +[ + { + "id": "worker-id-stability", + "category": "ci_failure_recurrence", + "documents": [ + { + "id": "session-1", + "timestamp": "2026-06-20T09:30:00Z", + "content": "Session 1: A container deployment used the hostname as the default worker id. The team saw pending consolidation operations that were not claimed after a restart. The accepted fix direction was to require a stable HINDSIGHT_API_WORKER_ID in container deployments and warn when the process falls back to a container hostname." + }, + { + "id": "session-2", + "timestamp": "2026-06-21T11:10:00Z", + "content": "Session 2: A follow-up noted that the warning must cover both API-embedded workers and standalone worker processes. Explicit --worker-id and HINDSIGHT_API_WORKER_ID should stay quiet; only the default hostname fallback should warn." + }, + { + "id": "session-3", + "timestamp": "2026-06-22T15:40:00Z", + "content": "Session 3: Regression tests should simulate a container hostname and verify three paths: warning on fallback, no warning for an explicit worker id, and no warning outside a container." + } + ], + "queries": [ + { + "id": "q1", + "category": "ci_failure_recurrence", + "query": "A new standalone worker test fails because it expects no warning when --worker-id is provided. What prior decision should guide the fix?", + "gold_document_ids": ["session-2", "session-3"], + "gold_answers": [ + "Explicit worker ids should not warn; only the default hostname fallback inside a container should warn, and the regression tests should cover explicit id, fallback warning, and non-container paths." + ] + } + ] + }, + { + "id": "retrieval-limit-wiring", + "category": "review_preference", + "documents": [ + { + "id": "session-1", + "timestamp": "2026-06-18T08:00:00Z", + "content": "Session 1: The benchmark runner uses provider retrieval results to build a RAG context. The review preference is to keep benchmark fixes narrow and avoid changing scoring, prompts, or provider APIs unless a bug requires it." + }, + { + "id": "session-2", + "timestamp": "2026-06-19T13:30:00Z", + "content": "Session 2: A bug was found in agentic-rag mode: the configured retrieval limit was parsed but not passed into the memory provider, so a mode using k=3 could still retrieve the provider default. The expected fix is to thread the limit through the existing call rather than add a new retrieval abstraction." + }, + { + "id": "session-3", + "timestamp": "2026-06-21T10:15:00Z", + "content": "Session 3: A maintainer asked for small benchmark PRs that isolate one behavior and include a local command that proves the path still runs." + } + ], + "queries": [ + { + "id": "q1", + "category": "review_preference", + "query": "When fixing another agentic-rag retrieval-count bug, what implementation style should be preferred?", + "gold_document_ids": ["session-1", "session-2", "session-3"], + "gold_answers": [ + "Keep the patch narrow: pass the existing retrieval limit through the current provider call, avoid unrelated scoring or prompt changes, and include a local command proving the path still runs." + ] + } + ] + }, + { + "id": "stale-docs-boundary", + "category": "release_or_docs_sync", + "documents": [ + { + "id": "session-1", + "timestamp": "2026-05-07T10:00:00Z", + "content": "Session 1: An old Windows setup note said Hindsight v0.6.0 was current and recommended checking /metrics to verify that the API was running." + }, + { + "id": "session-2", + "timestamp": "2026-06-18T09:36:01Z", + "content": "Session 2: Release v0.8.3 became the current Hindsight release. Setup notes that mention v0.6.0 as current should be treated as stale unless they are explicitly historical." + }, + { + "id": "session-3", + "timestamp": "2026-06-20T08:44:00Z", + "content": "Session 3: For Chinese-network Windows installs, users reported that HF_ENDPOINT can help local embedding downloads, but FlashRank may need a direct proxy or manual cache download if the mirror does not contain the model." + } + ], + "queries": [ + { + "id": "q1", + "category": "release_or_docs_sync", + "query": "A docs reply asks whether v0.6.0 is still the current recommended version for Windows setup. What should the answer do with the old memory?", + "gold_document_ids": ["session-1", "session-2"], + "gold_answers": [ + "It should treat the v0.6.0 note as stale because v0.8.3 is current, while preserving any still-useful setup troubleshooting as historical context." + ] + } + ] + } +] diff --git a/src/memory_bench/dataset/__init__.py b/src/memory_bench/dataset/__init__.py index 599bc54..4c58724 100644 --- a/src/memory_bench/dataset/__init__.py +++ b/src/memory_bench/dataset/__init__.py @@ -1,5 +1,6 @@ from .base import Dataset from .beam import BEAMDataset +from .codingagent import CodingAgentDataset from .lifebench import LifeBenchDataset from .locomo import LoComoDataset from .longmemeval import LongMemEvalDataset @@ -9,6 +10,7 @@ REGISTRY: dict[str, type[Dataset]] = { "beam": BEAMDataset, + "codingagent": CodingAgentDataset, "lifebench": LifeBenchDataset, "locomo": LoComoDataset, "longmemeval": LongMemEvalDataset, diff --git a/src/memory_bench/dataset/codingagent.py b/src/memory_bench/dataset/codingagent.py new file mode 100644 index 0000000..1284957 --- /dev/null +++ b/src/memory_bench/dataset/codingagent.py @@ -0,0 +1,138 @@ +""" +Synthetic coding-agent memory dataset. + +This small seed split tests whether a memory system helps with engineering +continuity across sessions: reusing prior decisions, avoiding stale facts, and +remembering CI/review constraints without loading a whole repo into context. +""" +import json +from pathlib import Path + +from rich.console import Console +from rich.table import Table + +from .base import Dataset +from ..models import Document, Query + +SPLITS = ["synthetic"] + +_CATEGORIES = [ + "ci_failure_recurrence", + "review_preference", + "release_or_docs_sync", +] + + +class CodingAgentDataset(Dataset): + """Small synthetic dataset for coding-agent memory behavior.""" + + name = "codingagent" + description = "Synthetic multi-session engineering traces for coding-agent memory." + splits = SPLITS + task_type = "open" + isolation_unit = "case" + links = [ + {"label": "Hindsight issue", "url": "https://github.com/vectorize-io/hindsight/issues/2347"}, + ] + + def _data_path(self, split: str) -> Path: + if split not in SPLITS: + raise ValueError(f"Unknown CodingAgent split '{split}'. Available: {SPLITS}") + return Path(__file__).parents[3] / "data" / "codingagent" / split / "cases.json" + + def _load_cases(self, split: str) -> list[dict]: + with open(self._data_path(split), encoding="utf-8") as f: + return json.load(f) + + def categories(self, split: str) -> list[str] | None: + return _CATEGORIES + + def category_type(self, split: str, category: str) -> str: + return "query" + + def get_result_categories(self, meta: dict) -> dict[str, list[str]]: + axes: dict[str, list[str]] = {} + if category := meta.get("category"): + axes["Category"] = [category] + if case_id := meta.get("case_id"): + axes["Case"] = [case_id] + return axes + + def load_queries( + self, + split: str, + category: str | None = None, + limit: int | None = None, + ) -> list[Query]: + queries: list[Query] = [] + for case in self._load_cases(split): + case_id = case["id"] + for raw_query in case.get("queries", []): + query_category = raw_query.get("category", case.get("category")) + if category and query_category != category: + continue + queries.append(Query( + id=f"{case_id}_{raw_query['id']}", + query=raw_query["query"], + gold_ids=[ + f"{case_id}_{doc_id}" + for doc_id in raw_query.get("gold_document_ids", []) + ], + gold_answers=raw_query.get("gold_answers", []), + user_id=case_id, + meta={"case_id": case_id, "category": query_category}, + )) + + if limit: + queries = queries[:limit] + return queries + + def load_documents( + self, + split: str, + category: str | None = None, + limit: int | None = None, + ids: set[str] | None = None, + user_ids: set[str] | None = None, + ) -> list[Document]: + documents: list[Document] = [] + for case in self._load_cases(split): + case_id = case["id"] + if user_ids is not None and case_id not in user_ids: + continue + for raw_doc in case.get("documents", []): + doc_id = f"{case_id}_{raw_doc['id']}" + if ids is not None and doc_id not in ids: + continue + documents.append(Document( + id=doc_id, + content=raw_doc["content"], + user_id=case_id, + timestamp=raw_doc.get("timestamp"), + context=f"codingagent:{case_id}", + )) + + if limit and ids is None: + documents = documents[:limit] + return documents + + def dataset_stats(self, console: Console, **_) -> None: + table = Table(title="CodingAgent synthetic dataset stats") + table.add_column("Split", style="bold") + table.add_column("Cases", justify="right") + table.add_column("Documents", justify="right") + table.add_column("Queries", justify="right") + table.add_column("Categories", justify="right") + + for split in SPLITS: + cases = self._load_cases(split) + documents = sum(len(case.get("documents", [])) for case in cases) + queries = sum(len(case.get("queries", [])) for case in cases) + categories = { + query.get("category", case.get("category")) + for case in cases + for query in case.get("queries", []) + } + table.add_row(split, str(len(cases)), str(documents), str(queries), str(len(categories))) + + console.print(table)