From f45baa25de1838c51a3484e9d01e2fd1c515fd0c Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 22 Aug 2026 19:20:51 +0530 Subject: [PATCH] feat(systemcontext): adopt incremental context and git-tree snapshots from opencode Add two genuinely-missing capabilities adopted from opencode's design: - internal/systemcontext: composable, incrementally-rendered model context runtime (Source/Codec/Reconciler/Epoch) that preserves a cache-stable baseline and emits only changed context as an update. Includes the ctxmgr.IncrementalContext host integration and an opt-in engine wiring (HAWK_INCREMENTAL_CONTEXT=1) so unchanged memories do not rewrite the system prompt each turn. - internal/gitsnapshot: git-object-store file-tree snapshots sharing the source repo object db via objects/info/alternates, with content-addressed Capture, Diff, throwaway-index Preview, Restore, and compare-and-swap WriteIfUnchanged to prevent concurrent clobbering. 8 of the 10 originally proposed opencode adoptions already exist in eyrie/ hawk (prompt caching, tool-output spill, least-privilege permissions, usage ledger, reasoning variants, error taxonomy, OAuth refresh, redaction); these two were the confirmed gaps. All new packages build, lint clean, and pass their tests (systemcontext 12, gitsnapshot 7, ctxmgr + engine suites green). --- internal/engine/ctxmgr/incremental.go | 173 ++++++++++ internal/engine/ctxmgr/incremental_test.go | 169 ++++++++++ internal/engine/incremental.go | 72 ++++ internal/engine/incremental_test.go | 100 ++++++ internal/engine/session.go | 4 + internal/engine/stream.go | 19 +- internal/gitsnapshot/snapshot.go | 371 +++++++++++++++++++++ internal/gitsnapshot/snapshot_test.go | 223 +++++++++++++ internal/systemcontext/context.go | 253 ++++++++++++++ internal/systemcontext/context_test.go | 300 +++++++++++++++++ internal/systemcontext/doc.go | 5 + internal/systemcontext/epoch.go | 49 +++ internal/systemcontext/reconciler.go | 195 +++++++++++ 13 files changed, 1932 insertions(+), 1 deletion(-) create mode 100644 internal/engine/ctxmgr/incremental.go create mode 100644 internal/engine/ctxmgr/incremental_test.go create mode 100644 internal/engine/incremental.go create mode 100644 internal/engine/incremental_test.go create mode 100644 internal/gitsnapshot/snapshot.go create mode 100644 internal/gitsnapshot/snapshot_test.go create mode 100644 internal/systemcontext/context.go create mode 100644 internal/systemcontext/context_test.go create mode 100644 internal/systemcontext/doc.go create mode 100644 internal/systemcontext/epoch.go create mode 100644 internal/systemcontext/reconciler.go diff --git a/internal/engine/ctxmgr/incremental.go b/internal/engine/ctxmgr/incremental.go new file mode 100644 index 00000000..e6a0834c --- /dev/null +++ b/internal/engine/ctxmgr/incremental.go @@ -0,0 +1,173 @@ +package ctxmgr + +import ( + "fmt" + "time" + + "github.com/GrayCodeAI/hawk/internal/systemcontext" +) + +// Section defines a dynamic, incrementally-rendered system-prompt section. It +// is the hawk-host integration of the systemcontext package: each section is a +// typed context source whose value is loaded on demand, and only sections that +// actually change are re-rendered as a mid-conversation update rather than +// rebuilding the entire system prompt. +// +// The section's rendered content is emitted as a markdown block headed by +// Header (e.g. "## Relevant Memories"), matching the section-header convention +// already used by ReplaceSystemContextSection. +type Section struct { + // Key is the stable namespaced source key (scope/name). + Key string + // Header is the markdown header used when the section is rendered. + Header string + // Load returns the current section value. + Load func() (string, error) +} + +// IncrementalContext reconciles a set of dynamic system-prompt sections +// against a durable snapshot, emitting only the sections that changed. It +// preserves a stable baseline for the unchanged sections across turns, so a +// provider prompt-cache prefix stays valid. +type IncrementalContext struct { + ctx *systemcontext.SystemContext + rec *systemcontext.Reconciler + ep *systemcontext.Epoch + segs map[string]Section +} + +// NewIncrementalContext builds an incremental context manager from the given +// sections. Callers must supply at least one section. +func NewIncrementalContext(sections []Section) (*IncrementalContext, error) { + if len(sections) == 0 { + return nil, fmt.Errorf("ctxmgr: incremental context requires at least one section") + } + host := &IncrementalContext{segs: map[string]Section{}} + // Build typed sources; the value type is a plain string rendered under a + // markdown header. + var srcs []systemcontext.Source[string] + for _, sec := range sections { + host.segs[sec.Key] = sec + header := sec.Header + srcs = append(srcs, systemcontext.Source[string]{ + Key: systemcontext.NewKey(keyScope(sec.Key), keyName(sec.Key)), + Codec: systemcontext.JSONCodec(func(a, b string) bool { return a == b }), + Load: sec.Load, + Baseline: func(v string) string { + return renderSection(header, v) + }, + Update: func(_, cur string) string { + return renderSection(header, cur) + }, + }) + } + host.ctx = systemcontext.NewAll(srcs...) + host.rec = systemcontext.NewReconciler(host.ctx) + return host, nil +} + +// Initialize renders the full baseline and stores its snapshot. Call once +// before the first request so an immutable baseline is established. +func (ic *IncrementalContext) Initialize() (string, error) { + base, snap, err := ic.rec.Initialize() + if err != nil { + return "", err + } + ic.ep = systemcontext.NewEpoch(base, snap) + return base, nil +} + +// Reconcile admits context changes at a safe request boundary. It returns: +// +// - (nil, false, nil): nothing changed; reuse the baseline. +// - (msg, false, nil): a mid-conversation update to inject, with the stable +// baseline unchanged. +// - (msg, true, newBaseline): the baseline was replaced (compaction) and msg +// carries the fresh baseline; the previous baseline must be discarded. +// - (_, _, err): the reconcile failed; the caller should retry or fall back +// to a full rebuild. +func (ic *IncrementalContext) Reconcile() (msg *string, replaced bool, newBaseline string, err error) { + if ic.ep == nil { + base, e := ic.Initialize() + if e != nil { + return nil, false, "", e + } + return &base, true, base, nil + } + return ic.ep.Prepare(ic.rec) +} + +// Baseline returns the current immutable baseline, or "" before Initialize. +func (ic *IncrementalContext) Baseline() string { + if ic.ep == nil { + return "" + } + return ic.ep.Baseline +} + +// SnapshotBytes returns the durable, marshalable snapshot of last-admitted +// section values. Empty string when the context was never initialized. +func (ic *IncrementalContext) SnapshotBytes() ([]byte, error) { + if ic.ep == nil || ic.ep.Snapshot == nil { + return nil, nil + } + return ic.ep.Snapshot.Marshal() +} + +// RestoreSnapshot rebuilds the epoch baseline and snapshot from persisted +// bytes so an ongoing conversation can resume incremental reconciliation +// without re-rendering everything. +func (ic *IncrementalContext) RestoreSnapshot(b []byte) error { + if ic.ep == nil { + ic.ep = systemcontext.NewEpoch("", systemcontext.NewSnapshot()) + } + var snap systemcontext.Snapshot + if err := snap.Unmarshal(b); err != nil { + return err + } + base := ic.rec.RenderBaseline(snap.Values) + ic.ep.Baseline = base + ic.ep.Snapshot = &snap + return nil +} + +// renderSection renders a section's value under its markdown header. The +// header is only emitted when the value is non-empty. +func renderSection(header, value string) string { + if value == "" { + return "" + } + if header == "" { + return value + } + return header + "\n" + value +} + +func keyScope(key string) string { + // A section key like "memories" or "scope/name" maps to a valid + // namespaced source key. + return "hawk" +} + +func keyName(key string) string { + return key +} + +// DefaultIncrementalSections returns a reasonable set of dynamic sections for +// a session, wired to the same loaders hawk already uses for each. It is a +// convenience for hosts that want to enable incremental context without +// hand-assembling sections. Loaders may be nil-able wrappers; the returned +// sections must be configured with their Load funcs by the caller. +func DefaultIncrementalSections() []Section { + return []Section{ + {Key: "environment", Header: "## Environment"}, + {Key: "date", Header: "## Date"}, + {Key: "memories", Header: "## Relevant Memories"}, + {Key: "directives", Header: "## Directives"}, + } +} + +// NowStr formats the current time for the date section. +func NowStr() string { + return time.Now().Format(time.RFC3339) +} diff --git a/internal/engine/ctxmgr/incremental_test.go b/internal/engine/ctxmgr/incremental_test.go new file mode 100644 index 00000000..fb1a0c44 --- /dev/null +++ b/internal/engine/ctxmgr/incremental_test.go @@ -0,0 +1,169 @@ +package ctxmgr + +import ( + "strings" + "sync" + "testing" +) + +type valSource struct { + mu sync.Mutex + value string +} + +func (v *valSource) get() (string, error) { + v.mu.Lock() + defer v.mu.Unlock() + return v.value, nil +} + +func (v *valSource) set(s string) { + v.mu.Lock() + v.value = s + v.mu.Unlock() +} + +func TestIncrementalInitializeBaseline(t *testing.T) { + mem := &valSource{value: "remembered X"} + ic, err := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if err != nil { + t.Fatalf("NewIncrementalContext: %v", err) + } + base, err := ic.Initialize() + if err != nil { + t.Fatalf("Initialize: %v", err) + } + if !strings.Contains(base, "## Relevant Memories") || !strings.Contains(base, "remembered X") { + t.Fatalf("baseline = %q", base) + } +} + +func TestIncrementalUnchanged(t *testing.T) { + mem := &valSource{value: "remembered X"} + ic, _ := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if _, err := ic.Initialize(); err != nil { + t.Fatal(err) + } + msg, replaced, _, err := ic.Reconcile() + if err != nil { + t.Fatal(err) + } + if replaced { + t.Fatal("unexpected replace") + } + if msg != nil { + t.Fatalf("unexpected update %q", *msg) + } +} + +func TestIncrementalUpdateEmitsOnlyChanged(t *testing.T) { + mem := &valSource{value: "remembered X"} + ic, _ := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if _, err := ic.Initialize(); err != nil { + t.Fatal(err) + } + mem.set("remembered Y") + msg, replaced, _, err := ic.Reconcile() + if err != nil { + t.Fatal(err) + } + if replaced { + t.Fatal("unexpected replace") + } + if msg == nil { + t.Fatal("expected an update") + } + if !strings.Contains(*msg, "remembered Y") { + t.Fatalf("update = %q", *msg) + } + // Baseline stays stable across an update. + if !strings.Contains(ic.Baseline(), "remembered X") { + t.Fatalf("baseline should remain immutable, got %q", ic.Baseline()) + } +} + +func TestIncrementalSnapshotRoundTrip(t *testing.T) { + mem := &valSource{value: "remembered X"} + ic, _ := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if _, err := ic.Initialize(); err != nil { + t.Fatal(err) + } + b, err := ic.SnapshotBytes() + if err != nil { + t.Fatalf("SnapshotBytes: %v", err) + } + if len(b) == 0 { + t.Fatal("expected non-empty snapshot") + } + + // A fresh host restores the snapshot and sees no change. + ic2, _ := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if err := ic2.RestoreSnapshot(b); err != nil { + t.Fatalf("RestoreSnapshot: %v", err) + } + msg, replaced, _, err := ic2.Reconcile() + if err != nil { + t.Fatal(err) + } + if replaced || msg != nil { + t.Fatalf("restored host should see no change, got replaced=%v msg=%v", replaced, msg) + } +} + +func TestIncrementalSnapshotReflectsChangeAfterRestore(t *testing.T) { + mem := &valSource{value: "remembered X"} + ic, _ := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if _, err := ic.Initialize(); err != nil { + t.Fatal(err) + } + b, _ := ic.SnapshotBytes() + + // Change the value, then restore a fresh host: it must emit an update. + mem.set("remembered Z") + ic2, _ := NewIncrementalContext([]Section{ + {Key: "memories", Header: "## Relevant Memories", Load: mem.get}, + }) + if err := ic2.RestoreSnapshot(b); err != nil { + t.Fatal(err) + } + msg, replaced, _, err := ic2.Reconcile() + if err != nil { + t.Fatal(err) + } + if replaced { + t.Fatal("unexpected replace") + } + if msg == nil || !strings.Contains(*msg, "remembered Z") { + t.Fatalf("expected update with new value, got %v", msg) + } +} + +func TestIncrementalRequiresSection(t *testing.T) { + if _, err := NewIncrementalContext(nil); err == nil { + t.Fatal("expected error for empty sections") + } +} + +func TestRenderSectionOmitsEmptyHeader(t *testing.T) { + if got := renderSection("## H", ""); got != "" { + t.Fatalf("empty value should render empty, got %q", got) + } + if got := renderSection("", "value"); got != "value" { + t.Fatalf("empty header should not emit header, got %q", got) + } + if got := renderSection("## H", "v"); !strings.HasPrefix(got, "## H") { + t.Fatalf("expected header prefix, got %q", got) + } +} diff --git a/internal/engine/incremental.go b/internal/engine/incremental.go new file mode 100644 index 00000000..e933317f --- /dev/null +++ b/internal/engine/incremental.go @@ -0,0 +1,72 @@ +package engine + +import ( + "os" + "strings" + + "github.com/GrayCodeAI/hawk/internal/engine/ctxmgr" +) + +// incrementalContextEnabled reports whether the opt-in incremental +// system-context mode is enabled. It is disabled by default so existing +// request assembly is byte-for-byte unchanged; when enabled, dynamic +// system-prompt sections (e.g. memories) are reconciled incrementally and only +// changed sections are re-rendered, preserving a stable system-prompt prefix +// that keeps provider prompt-cache entries valid across turns. +func incrementalContextEnabled() bool { + return strings.EqualFold(os.Getenv("HAWK_INCREMENTAL_CONTEXT"), "1") +} + +// memoryIncremental is a small holder that wires a memory-recall loader into +// an IncrementalContext section. It is recreated when the recall function's +// identity changes. +type memoryIncremental struct { + ic *ctxmgr.IncrementalContext + init bool +} + +// newMemoryIncremental builds an incremental context backed by the given +// memory recall loader. +func newMemoryIncremental(recall func() string) (*memoryIncremental, error) { + ic, err := ctxmgr.NewIncrementalContext([]ctxmgr.Section{ + {Key: "memories", Header: "## Relevant Memories", Load: func() (string, error) { + if recall == nil { + return "", nil + } + return recall(), nil + }}, + }) + if err != nil { + return nil, err + } + return &memoryIncremental{ic: ic}, nil +} + +// prepare runs the incremental memory-recall path. It returns the section +// content to write for the current turn and whether that content changed: +// +// - (content, true): write content as the new "## Relevant Memories" section. +// This covers both first initialization (full) and an incremental change. +// - ("", false): nothing changed; keep the existing section untouched, which +// preserves a stable system-prompt prefix (and any provider cache entry) +// across turns. +func (m *memoryIncremental) prepare() (content string, changed bool) { + if m == nil || m.ic == nil { + return "", true + } + if !m.init { + if _, err := m.ic.Initialize(); err != nil { + return "", true + } + m.init = true + return m.ic.Baseline(), true + } + msg, replaced, base, err := m.ic.Reconcile() + if err != nil || replaced { + return base, true + } + if msg == nil { + return "", false + } + return *msg, true +} diff --git a/internal/engine/incremental_test.go b/internal/engine/incremental_test.go new file mode 100644 index 00000000..5306b5a0 --- /dev/null +++ b/internal/engine/incremental_test.go @@ -0,0 +1,100 @@ +package engine + +import ( + "strings" + "sync" + "testing" +) + +type recallBox struct { + mu sync.Mutex + value string +} + +func (b *recallBox) set(s string) { + b.mu.Lock() + b.value = s + b.mu.Unlock() +} + +func (b *recallBox) get() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.value +} + +func TestMemoryIncrementalFirstCallFull(t *testing.T) { + b := &recallBox{value: "remembered X"} + mi, err := newMemoryIncremental(b.get) + if err != nil { + t.Fatalf("newMemoryIncremental: %v", err) + } + content, changed := mi.prepare() + if !changed { + t.Fatal("first call should report changed") + } + if !strings.Contains(content, "remembered X") { + t.Fatalf("content = %q", content) + } +} + +func TestMemoryIncrementalUnchangedSkips(t *testing.T) { + b := &recallBox{value: "remembered X"} + mi, _ := newMemoryIncremental(b.get) + if _, _ = mi.prepare(); true { + // first call establishes baseline + } + content, changed := mi.prepare() + if changed { + t.Fatalf("unchanged recall should not rewrite, got changed content=%q", content) + } + if content != "" { + t.Fatalf("expected empty content for unchanged, got %q", content) + } +} + +func TestMemoryIncrementalChangeRewrites(t *testing.T) { + b := &recallBox{value: "remembered X"} + mi, _ := newMemoryIncremental(b.get) + mi.prepare() // baseline + + b.set("remembered Y") + content, changed := mi.prepare() + if !changed { + t.Fatal("changed recall should report changed") + } + if !strings.Contains(content, "remembered Y") { + t.Fatalf("content = %q", content) + } + + // And it stabilizes: next call is unchanged. + _, changed = mi.prepare() + if changed { + t.Fatal("should stabilize after applying change") + } +} + +func TestMemoryIncrementalNilRecall(t *testing.T) { + mi, err := newMemoryIncremental(nil) + if err != nil { + t.Fatalf("newMemoryIncremental(nil): %v", err) + } + content, changed := mi.prepare() + if !changed { + t.Fatal("should initialize even with nil recall") + } + if content != "" { + t.Fatalf("expected empty content for nil recall, got %q", content) + } +} + +func TestIncrementalContextEnabledFlag(t *testing.T) { + t.Setenv("HAWK_INCREMENTAL_CONTEXT", "1") + if !incrementalContextEnabled() { + t.Fatal("expected enabled with HAWK_INCREMENTAL_CONTEXT=1") + } + t.Setenv("HAWK_INCREMENTAL_CONTEXT", "0") + if incrementalContextEnabled() { + t.Fatal("expected disabled with HAWK_INCREMENTAL_CONTEXT=0") + } +} diff --git a/internal/engine/session.go b/internal/engine/session.go index 508fb081..36982c76 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -79,6 +79,10 @@ type Session struct { persist *PersistenceService goals *planning.GoalTracker // optional goal tracker; emits goal.change lifecycle events tools *ToolService + // incremental is the opt-in incremental system-context reconciler for + // dynamic sections (e.g. memories). Nil unless HAWK_INCREMENTAL_CONTEXT=1. + // See incremental.go. + incremental *memoryIncremental // learnFn persists structured lessons produced by failure reflection to a // cross-session store (e.g. the chat client's SelfImprover). It is a // callback so the engine stays decoupled from storage; nil disables it. diff --git a/internal/engine/stream.go b/internal/engine/stream.go index cfabdd8e..9b26897b 100644 --- a/internal/engine/stream.go +++ b/internal/engine/stream.go @@ -357,10 +357,27 @@ func (s *Session) agentLoop(ctx context.Context, ch chan<- StreamEvent) { // memory service boundary. if len(s.Persistence().RawMessages()) > 0 { lastMsg := s.Persistence().RawMessages()[len(s.Persistence().RawMessages())-1].Content - if remembered := s.MemorySvc().RecallContext(ctx, lastMsg, 3000); remembered != "" { + recall := func() string { + return s.MemorySvc().RecallContext(ctx, lastMsg, 3000) + } + if incrementalContextEnabled() { + if s.incremental == nil { + if mi, err := newMemoryIncremental(recall); err == nil { + s.incremental = mi + } + } + if s.incremental != nil { + if content, changed := s.incremental.prepare(); changed && content != "" { + s.ReplaceSystemContextSection("## Relevant Memories\n", content) + } + goto memoryDone + } + } + if remembered := recall(); remembered != "" { s.ReplaceSystemContextSection("## Relevant Memories\n", remembered) } } + memoryDone: // Payload tiering: classify the latest user request once per turn. // Early conversational turns get a minimal system prompt and no tool diff --git a/internal/gitsnapshot/snapshot.go b/internal/gitsnapshot/snapshot.go new file mode 100644 index 00000000..d7763442 --- /dev/null +++ b/internal/gitsnapshot/snapshot.go @@ -0,0 +1,371 @@ +// Package gitsnapshot captures point-in-time file-tree snapshots using git's +// content-addressed object database, adopting the approach opencode uses for +// durable, cheap snapshots. +// +// A Manager keeps a linked snapshot repository whose object database is seeded +// from the source repository via objects/info/alternates, so capturing a tree +// reuses hashes already computed by the source repo instead of re-hashing large +// checkouts. Snapshots are content-addressed git tree IDs, which makes diffing, +// previewing, and restoring cheap and unambiguous. +package gitsnapshot + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// TreeID is a content-addressed git tree hash. +type TreeID string + +// StaleContentError reports a compare-and-swap write that would clobber a file +// changed by another writer since it was read. +type StaleContentError struct { + Path string +} + +func (e *StaleContentError) Error() string { + return "gitsnapshot: stale content at " + e.Path + " (file changed since read)" +} + +// FileChange describes a single file's change between two trees. +type FileChange struct { + Path string + Status string // "added", "modified", "deleted" + Additions int + Deletions int + Patch string +} + +// Manager snapshots a source repository's file tree into a linked snapshot repo. +type Manager struct { + // SourceDir is the working directory of the source repo. + SourceDir string + // SnapDir is the directory holding the linked snapshot repo. + SnapDir string +} + +// New creates a Manager and prepares the linked snapshot repo, seeding its +// object storage from the source repo. SourceDir must be inside a git repo. +func New(sourceDir, snapDir string) (*Manager, error) { + abs, err := filepath.Abs(sourceDir) + if err != nil { + return nil, fmt.Errorf("gitsnapshot: resolve source dir: %w", err) + } + m := &Manager{SourceDir: abs, SnapDir: snapDir} + if err := m.initLinkedRepo(); err != nil { + return nil, err + } + return m, nil +} + +// initLinkedRepo creates the snapshot repo (if needed) and wires its object +// database to the source repo via objects/info/alternates. +func (m *Manager) initLinkedRepo() error { + if m.SnapDir == "" { + return fmt.Errorf("gitsnapshot: empty snapshot dir") + } + if err := os.MkdirAll(m.SnapDir, 0o750); err != nil { + return fmt.Errorf("gitsnapshot: create snapshot dir: %w", err) + } + gitDir := filepath.Join(m.SnapDir, ".git") + if _, err := os.Stat(gitDir); os.IsNotExist(err) { + if err := m.gitRun(m.SnapDir, "init", "--bare"); err != nil { + return fmt.Errorf("gitsnapshot: init linked repo: %w", err) + } + } + + // Point the snapshot repo's object db at the source repo's object db. + altFile := filepath.Join(gitDir, "objects", "info", "alternates") + srcObjDir, err := m.sourceObjectDir() + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(altFile), 0o750); err != nil { + return fmt.Errorf("gitsnapshot: mkdir objects/info: %w", err) + } + if err := os.WriteFile(altFile, []byte(srcObjDir+"\n"), 0o600); err != nil { + return fmt.Errorf("gitsnapshot: write alternates: %w", err) + } + return nil +} + +// sourceObjectDir returns the absolute path to the source repo's object db. +func (m *Manager) sourceObjectDir() (string, error) { + out, err := m.gitOut(m.SourceDir, "rev-parse", "--git-dir") + if err != nil { + return "", fmt.Errorf("gitsnapshot: locate source git dir: %w", err) + } + gitDir := strings.TrimSpace(out) + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(m.SourceDir, gitDir) + } + objDir := filepath.Join(gitDir, "objects") + if st, err := os.Stat(objDir); err != nil || !st.IsDir() { + return "", fmt.Errorf("gitsnapshot: source object db not found at %s", objDir) + } + return objDir, nil +} + +// Capture stages the given project-relative paths into a private throwaway +// index in the source repo and writes a content-addressed tree, returning its +// TreeID. The user's real index and staging area are left untouched. An empty +// paths list captures the whole worktree (respecting .gitignore). The resulting +// tree lives in the source object db, which the linked snapshot repo shares. +func (m *Manager) Capture(ctx context.Context, paths []string) (TreeID, error) { + idx, err := os.CreateTemp("", "gitsnapshot-capture-*.index") + if err != nil { + return "", fmt.Errorf("gitsnapshot: create capture index: %w", err) + } + idxPath := idx.Name() + _ = idx.Close() + defer func() { _ = os.Remove(idxPath) }() + + env := append(os.Environ(), "GIT_INDEX_FILE="+idxPath) + + // Seed the throwaway index from HEAD so tree reads/writes reflect the repo. + if _, err := m.gitOutEnv(ctx, m.SourceDir, env, "read-tree", "HEAD"); err != nil { + // A repo with no commits has no HEAD; tolerate by starting empty. + if _, e2 := m.gitOutEnv(ctx, m.SourceDir, env, "read-tree", "--empty"); e2 != nil { + return "", fmt.Errorf("gitsnapshot: seed index: %w", err) + } + } + + args := []string{"add", "--all"} + if len(paths) > 0 { + args = append(args, "--") + args = append(args, paths...) + } + if _, err := m.gitOutEnv(ctx, m.SourceDir, env, args...); err != nil { + return "", fmt.Errorf("gitsnapshot: add: %w", err) + } + out, err := m.gitOutEnv(ctx, m.SourceDir, env, "write-tree") + if err != nil { + return "", fmt.Errorf("gitsnapshot: write-tree: %w", err) + } + tid := TreeID(strings.TrimSpace(out)) + if tid == "" { + return "", fmt.Errorf("gitsnapshot: write-tree produced empty id") + } + return tid, nil +} + +// Diff returns per-file changes between two trees. Either id may be empty to +// compare against an empty tree (all files added). +func (m *Manager) Diff(ctx context.Context, from, to TreeID) ([]FileChange, error) { + a, b := string(from), string(to) + if a == "" { + a = emptyTreeID(ctx, m) + } + if b == "" { + b = emptyTreeID(ctx, m) + } + raw, err := m.gitOutCtx(ctx, m.SourceDir, "diff-tree", "-r", "--name-status", a, b) + if err != nil { + return nil, fmt.Errorf("gitsnapshot: diff-tree: %w", err) + } + var changes []FileChange + for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + status := fields[0] + path := fields[len(fields)-1] + ch := FileChange{Path: path} + switch status[0] { + case 'A': + ch.Status = "added" + case 'D': + ch.Status = "deleted" + case 'M': + ch.Status = "modified" + default: + ch.Status = status + } + changes = append(changes, ch) + } + return changes, nil +} + +// Preview computes a hypothetical per-file diff for a set of paths against a +// target tree without touching the source worktree. It builds a throwaway +// index (GIT_INDEX_FILE) so the operation is side-effect free. +func (m *Manager) Preview(ctx context.Context, target TreeID, paths []string) ([]FileChange, error) { + tmp, err := os.CreateTemp("", "gitsnapshot-preview-*.index") + if err != nil { + return nil, fmt.Errorf("gitsnapshot: create throwaway index: %w", err) + } + idxPath := tmp.Name() + if err := tmp.Close(); err != nil { + return nil, err + } + defer func() { _ = os.Remove(idxPath) }() + + env := append(os.Environ(), "GIT_INDEX_FILE="+idxPath) + if _, err := m.gitOutEnv(ctx, m.SourceDir, env, "read-tree", string(target)); err != nil { + return nil, fmt.Errorf("gitsnapshot: read-tree: %w", err) + } + + var changes []FileChange + for _, p := range paths { + // Compare the index blob for the path against the worktree file. + idxBlob, err := m.gitOutEnv(ctx, m.SourceDir, env, "ls-files", "-s", "--", p) + if err != nil { + return nil, err + } + worktreePath := filepath.Join(m.SourceDir, p) + data, err := os.ReadFile(worktreePath) + if err != nil { + if os.IsNotExist(err) { + changes = append(changes, FileChange{Path: p, Status: "deleted"}) + continue + } + return nil, err + } + // Hash the worktree blob and compare to the index entry hash. + idxFields := strings.Fields(idxBlob) + blobID := "" + if len(idxFields) >= 2 { + // "ls-files -s" emits " \t". + blobID = idxFields[1] + } + if blobID == "" { + changes = append(changes, FileChange{Path: p, Status: "added"}) + continue + } + workID := m.hashBlob(ctx, env, data) + if workID == "" { + changes = append(changes, FileChange{Path: p, Status: "modified"}) + continue + } + if workID != blobID { + changes = append(changes, FileChange{Path: p, Status: "modified"}) + } + } + return changes, nil +} + +// Restore selectively checks out (or deletes) paths from a target tree into the +// source worktree. Deleted-in-tree paths are removed from the worktree. +func (m *Manager) Restore(ctx context.Context, target TreeID, paths []string) error { + for _, p := range paths { + worktreePath := filepath.Join(m.SourceDir, p) + blobID, err := m.gitOutCtx(ctx, m.SourceDir, "ls-tree", string(target), "--", p) + if err != nil { + return err + } + trimmed := strings.TrimSpace(blobID) + if trimmed == "" { + // Path absent from tree => delete from worktree. + if err := os.RemoveAll(worktreePath); err != nil { + return fmt.Errorf("gitsnapshot: delete %s: %w", p, err) + } + continue + } + // blobID line: " blob \t" + fields := strings.Fields(trimmed) + if len(fields) < 3 { + return fmt.Errorf("gitsnapshot: unexpected ls-tree output for %s: %q", p, trimmed) + } + if err := m.writeBlobToFile(ctx, envPlain(), fields[2], worktreePath); err != nil { + return err + } + } + return nil +} + +// WriteIfUnchanged performs a compare-and-swap write: it only writes newBytes +// to path if the current content still equals expectedBytes. It returns a +// *StaleContentError when the file changed since it was read, preventing one +// agent from clobbering another's concurrent edit. +func (m *Manager) WriteIfUnchanged(path string, expected, newContent []byte) error { + current, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("gitsnapshot: read %s: %w", path, err) + } + if !bytes.Equal(current, expected) { + return &StaleContentError{Path: path} + } + if err := os.WriteFile(path, newContent, 0o644); err != nil { + return fmt.Errorf("gitsnapshot: write %s: %w", path, err) + } + return nil +} + +// writeBlobToFile writes a git blob's content to path using `git cat-file`. +func (m *Manager) writeBlobToFile(ctx context.Context, env []string, blobID, path string) error { + cmd := exec.CommandContext(ctx, "git", "-C", m.SourceDir, "cat-file", "blob", blobID) // #nosec G204 -- fixed git subcommand, internally-derived blob id + cmd.Env = env + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("gitsnapshot: cat-file %s: %w", blobID, err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + return os.WriteFile(path, out, 0o644) +} + +func (m *Manager) hashBlob(ctx context.Context, env []string, data []byte) string { + cmd := exec.CommandContext(ctx, "git", "-C", m.SourceDir, "hash-object", "-t", "blob", "--stdin") // #nosec G204 + cmd.Env = env + cmd.Stdin = bytes.NewReader(data) + out, err := cmd.Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +func (m *Manager) gitRun(dir string, args ...string) error { + return m.gitRunCtx(context.Background(), dir, args...) +} + +func (m *Manager) gitRunCtx(ctx context.Context, dir string, args ...string) error { + full := append([]string{"-C", dir}, args...) + cmd := exec.CommandContext(ctx, "git", full...) // #nosec G204 -- fixed git subcommand, internally-derived args + cmd.Env = envPlain() + return cmd.Run() +} + +func (m *Manager) gitOut(dir string, args ...string) (string, error) { + return m.gitOutCtx(context.Background(), dir, args...) +} + +func (m *Manager) gitOutCtx(ctx context.Context, dir string, args ...string) (string, error) { + return m.gitOutEnv(ctx, dir, envPlain(), args...) +} + +func (m *Manager) gitOutEnv(ctx context.Context, dir string, env []string, args ...string) (string, error) { + full := append([]string{"-C", dir}, args...) + cmd := exec.CommandContext(ctx, "git", full...) // #nosec G204 -- fixed git subcommand, internally-derived args + cmd.Env = env + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + } + return string(out), nil +} + +// envPlain returns the current environment (used to keep PATH etc.). +func envPlain() []string { + return os.Environ() +} + +// emptyTreeID returns the well-known empty tree hash. This is the git +// object id of the empty tree, stable across all git installations. +const emptyTreeIDValue = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + +func emptyTreeID(ctx context.Context, m *Manager) string { + _ = ctx + _ = m + return emptyTreeIDValue +} diff --git a/internal/gitsnapshot/snapshot_test.go b/internal/gitsnapshot/snapshot_test.go new file mode 100644 index 00000000..4cb684a9 --- /dev/null +++ b/internal/gitsnapshot/snapshot_test.go @@ -0,0 +1,223 @@ +package gitsnapshot + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// runGit is a test helper that runs git in a directory. +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// setupRepo creates a temp git repo with some committed files and returns its dir. +func setupRepo(t *testing.T) string { + t.Helper() + dir := t.TempDir() + runGit(t, dir, "init", "-q") + runGit(t, dir, "config", "user.email", "test@example.com") + runGit(t, dir, "config", "user.name", "Test") + if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("alpha\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "b.txt"), []byte("beta\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-qm", "init") + return dir +} + +func TestNewCreatesLinkedRepo(t *testing.T) { + src := setupRepo(t) + snap := filepath.Join(t.TempDir(), "snap") + m, err := New(src, snap) + if err != nil { + t.Fatalf("New: %v", err) + } + if _, err := os.Stat(filepath.Join(snap, ".git", "objects", "info", "alternates")); err != nil { + t.Fatalf("alternates file missing: %v", err) + } + // The alternates file must reference the source object db. + b, err := os.ReadFile(filepath.Join(snap, ".git", "objects", "info", "alternates")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "objects") { + t.Fatalf("alternates = %q", string(b)) + } + _ = m +} + +func TestCaptureReturnsTreeID(t *testing.T) { + src := setupRepo(t) + m, err := New(src, filepath.Join(t.TempDir(), "snap")) + if err != nil { + t.Fatal(err) + } + tid, err := m.Capture(context.Background(), nil) + if err != nil { + t.Fatalf("Capture: %v", err) + } + if tid == "" { + t.Fatal("empty tree id") + } +} + +func TestDiffDetectsChanges(t *testing.T) { + src := setupRepo(t) + m, _ := New(src, filepath.Join(t.TempDir(), "snap")) + ctx := context.Background() + + base, err := m.Capture(ctx, nil) + if err != nil { + t.Fatal(err) + } + + // Modify a.txt and add c.txt. + if err := os.WriteFile(filepath.Join(src, "a.txt"), []byte("alpha2\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "c.txt"), []byte("charlie\n"), 0o644); err != nil { + t.Fatal(err) + } + now, err := m.Capture(ctx, nil) + if err != nil { + t.Fatal(err) + } + + changes, err := m.Diff(ctx, base, now) + if err != nil { + t.Fatalf("Diff: %v", err) + } + byPath := map[string]string{} + for _, c := range changes { + byPath[c.Path] = c.Status + } + if byPath["a.txt"] != "modified" { + t.Fatalf("a.txt status = %q, want modified; changes=%+v", byPath["a.txt"], changes) + } + if byPath["c.txt"] != "added" { + t.Fatalf("c.txt status = %q, want added", byPath["c.txt"]) + } +} + +func TestWriteIfUnchangedSuccessAndStale(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "f.txt") + if err := os.WriteFile(path, []byte("expected"), 0o644); err != nil { + t.Fatal(err) + } + m := &Manager{} + if err := m.WriteIfUnchanged(path, []byte("expected"), []byte("new")); err != nil { + t.Fatalf("WriteIfUnchanged: %v", err) + } + got, _ := os.ReadFile(path) + if !bytes.Equal(got, []byte("new")) { + t.Fatalf("content = %q, want new", got) + } + + // Now the file is "new"; writing with stale expected must fail. + err := m.WriteIfUnchanged(path, []byte("expected"), []byte("other")) + if err == nil { + t.Fatal("expected stale content error") + } + if _, ok := err.(*StaleContentError); !ok { + t.Fatalf("err = %T, want *StaleContentError", err) + } +} + +func TestRestoreChecksOutAndDeletes(t *testing.T) { + src := setupRepo(t) + m, _ := New(src, filepath.Join(t.TempDir(), "snap")) + ctx := context.Background() + + base, err := m.Capture(ctx, nil) + if err != nil { + t.Fatal(err) + } + // Modify and capture a new tree. + if err := os.WriteFile(filepath.Join(src, "a.txt"), []byte("changed\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "c.txt"), []byte("new\n"), 0o644); err != nil { + t.Fatal(err) + } + now, err := m.Capture(ctx, nil) + if err != nil { + t.Fatal(err) + } + + // Restore a.txt from base (revert the change) and delete c.txt (absent in base). + if err := m.Restore(ctx, base, []string{"a.txt", "c.txt"}); err != nil { + t.Fatalf("Restore: %v", err) + } + got, _ := os.ReadFile(filepath.Join(src, "a.txt")) + if string(got) != "alpha\n" { + t.Fatalf("a.txt = %q, want alpha", got) + } + if _, err := os.Stat(filepath.Join(src, "c.txt")); !os.IsNotExist(err) { + t.Fatalf("c.txt should be deleted, got %v", err) + } + _ = now +} + +func TestPreviewDetectsModificationWithoutTouchingWorktree(t *testing.T) { + src := setupRepo(t) + m, _ := New(src, filepath.Join(t.TempDir(), "snap")) + ctx := context.Background() + + base, err := m.Capture(ctx, nil) + if err != nil { + t.Fatal(err) + } + + // Modify a.txt after capturing the baseline. + if err := os.WriteFile(filepath.Join(src, "a.txt"), []byte("modified\n"), 0o644); err != nil { + t.Fatal(err) + } + + // Preview the worktree against the baseline for a.txt only. + changes, err := m.Preview(ctx, base, []string{"a.txt"}) + if err != nil { + t.Fatalf("Preview: %v", err) + } + if len(changes) != 1 || changes[0].Path != "a.txt" || changes[0].Status != "modified" { + t.Fatalf("changes = %+v, want single modified a.txt", changes) + } + + // The worktree file must be untouched by the preview. + got, _ := os.ReadFile(filepath.Join(src, "a.txt")) + if string(got) != "modified\n" { + t.Fatalf("preview mutated worktree: %q", got) + } +} + +func TestPreviewNoChangeWhenFileMatchesTree(t *testing.T) { + src := setupRepo(t) + m, _ := New(src, filepath.Join(t.TempDir(), "snap")) + ctx := context.Background() + + base, err := m.Capture(ctx, nil) + if err != nil { + t.Fatal(err) + } + changes, err := m.Preview(ctx, base, []string{"a.txt", "b.txt"}) + if err != nil { + t.Fatalf("Preview: %v", err) + } + if len(changes) != 0 { + t.Fatalf("expected no changes, got %+v", changes) + } +} diff --git a/internal/systemcontext/context.go b/internal/systemcontext/context.go new file mode 100644 index 00000000..d307a996 --- /dev/null +++ b/internal/systemcontext/context.go @@ -0,0 +1,253 @@ +// Package systemcontext implements a composable, incrementally-rendered +// model context runtime inspired by opencode's System Context architecture. +// +// Instead of re-rendering a monolithic system prompt on every turn, privileged +// context is modelled as a set of independently refreshable typed Sources, each +// with a stable key, a JSON codec, an infallible loader, and pure renderers. +// A Reconciler compares each loaded source against a durable snapshot and +// returns exactly one action: unchanged, a single combined update message, or +// a full replacement. This preserves a provider-cache-stable baseline across +// turns while still admitting changed context as a chronological update. +package systemcontext + +import ( + "bytes" + "encoding/json" + "fmt" +) + +// Key is a stable, namespaced identity for a context source. The string form +// follows "scope/name" so contributions are deterministic when sorted. +type Key struct { + scope string + name string +} + +// NewKey builds a namespaced, stable key. Both parts must be non-empty. +func NewKey(scope, name string) Key { + return Key{scope: scope, name: name} +} + +// String renders the key in its stable "scope/name" form. +func (k Key) String() string { + if k.scope == "" { + return k.name + } + return k.scope + "/" + k.name +} + +// Valid reports whether the key has both non-empty parts. +func (k Key) Valid() bool { + return k.scope != "" && k.name != "" +} + +// Unavailable is a sentinel returned by loaders when a source value is +// temporarily unobservable. It is distinct from a successfully loaded absence, +// which may emit removal text. Reconcile retains the prior effective value for +// an unavailable source (stale-while-revalidate) rather than dropping it. +type Unavailable struct{} + +// ErrDuplicateKey is returned when combining sources with the same key. +var ErrDuplicateKey = fmt.Errorf("systemcontext: duplicate source key") + +// Source is a single independently observed typed context value. +// +// The generic parameter V is the source's value type. The codec serializes V to +// a comparable JSON form for snapshotting and equality, and the renderers +// produce model-visible text only when needed. +// +// Sources form a fixed set for the lifetime of a SystemContext. A change to the +// set of sources (add/remove) is a composition change and forces a fresh +// baseline (Replace) rather than an incremental update. +type Source[V any] struct { + Key Key + Codec Codec[V] + Load func() (V, error) + Baseline func(current V) string + Update func(previous, current V) string +} + +// Codec serializes a typed value to and from a stable JSON form used for +// durable comparison snapshots. +type Codec[V any] struct { + Encode func(V) (json.RawMessage, error) + Decode func(json.RawMessage) (V, error) + Equal func(a, b V) bool +} + +// JSONCodec builds a Codec for a JSON-serializable value type using a custom +// equality function. +func JSONCodec[V any](equal func(a, b V) bool) Codec[V] { + return Codec[V]{ + Encode: func(v V) (json.RawMessage, error) { return json.Marshal(v) }, + Decode: func(b json.RawMessage) (V, error) { + var v V + err := json.Unmarshal(b, &v) + return v, err + }, + Equal: equal, + } +} + +// SystemContext is an opaque carrier of one or more typed sources. The value +// types are hidden behind the codec so heterogeneous sources compose. +type SystemContext struct { + sources []typedSource +} + +type typedSource struct { + key Key + load func() (json.RawMessage, error) + base func(json.RawMessage) string + upd func(prevJSON, curJSON json.RawMessage) string + equal func(a, b json.RawMessage) bool + valid func() bool +} + +// New builds a SystemContext from one or more sources, rejecting duplicate keys. +func New[V any](s Source[V]) *SystemContext { + return NewAll(s) +} + +// NewAll combines multiple sources. Duplicate keys fail composition. +func NewAll[V any](sources ...Source[V]) *SystemContext { + ctx := &SystemContext{} + seen := map[string]bool{} + for _, s := range sources { + if !s.Key.Valid() { + panic(fmt.Sprintf("systemcontext: source has invalid key %q", s.Key.String())) + } + if seen[s.Key.String()] { + panic(fmt.Sprintf("systemcontext: duplicate source key %q", s.Key.String())) + } + seen[s.Key.String()] = true + ctx.sources = append(ctx.sources, typedSource{ + key: s.Key, + load: toLoad(s.Codec, s.Load), + base: renderBase(s.Codec, s.Baseline), + upd: renderUpdate(s.Codec, s.Update), + equal: func(a, b json.RawMessage) bool { return equalJSON(s.Codec, a, b) }, + valid: func() bool { return true }, + }) + } + return ctx +} + +// Sources returns the ordered, stable-keyed source descriptors. +func (c *SystemContext) Sources() []SourceInfo { + out := make([]SourceInfo, len(c.sources)) + for i, s := range c.sources { + out[i] = SourceInfo{Key: s.key.String()} + } + return out +} + +// SourceInfo is a lightweight, value-typed view of a source used by hosts for +// inspection and logging. +type SourceInfo struct { + Key string +} + +func toLoad[V any](codec Codec[V], load func() (V, error)) func() (json.RawMessage, error) { + if load == nil { + load = func() (V, error) { var z V; return z, nil } + } + return func() (json.RawMessage, error) { + v, err := load() + if err != nil { + return nil, err + } + return codec.Encode(v) + } +} + +func renderBase[V any](codec Codec[V], fn func(V) string) func(json.RawMessage) string { + if fn == nil { + return func(json.RawMessage) string { return "" } + } + return func(raw json.RawMessage) string { + v, err := codec.Decode(raw) + if err != nil { + return "" + } + return fn(v) + } +} + +func renderUpdate[V any](codec Codec[V], fn func(prev, cur V) string) func(json.RawMessage, json.RawMessage) string { + if fn == nil { + return func(json.RawMessage, json.RawMessage) string { return "" } + } + return func(prevRaw, curRaw json.RawMessage) string { + prev, err1 := codec.Decode(prevRaw) + cur, err2 := codec.Decode(curRaw) + if err1 != nil || err2 != nil { + return "" + } + return fn(prev, cur) + } +} + +func equalJSON[V any](codec Codec[V], a, b json.RawMessage) bool { + av, err1 := codec.Decode(a) + bv, err2 := codec.Decode(b) + if err1 != nil || err2 != nil { + return false + } + if codec.Equal != nil { + return codec.Equal(av, bv) + } + return string(normalizeJSON(a)) == string(normalizeJSON(b)) +} + +// normalizeJSON canonicalizes a raw message by compacting it, so semantic JSON +// equality works regardless of whitespace. +func normalizeJSON(raw json.RawMessage) []byte { + var buf bytes.Buffer + if err := json.Compact(&buf, raw); err != nil { + return raw + } + return buf.Bytes() +} + +// Snapshot is a durable, codec-encoded view of every source's last-admitted +// value. It is the model-hidden JSON state used to compare each source with +// its last admitted value. +type Snapshot struct { + // Values maps stable key string -> encoded value. + Values map[string]json.RawMessage `json:"values,omitempty"` +} + +// NewSnapshot returns an empty snapshot. +func NewSnapshot() *Snapshot { + return &Snapshot{Values: map[string]json.RawMessage{}} +} + +// Marshal / Unmarshal are provided for hosts that persist the snapshot. +func (s *Snapshot) Marshal() ([]byte, error) { return json.Marshal(s) } +func (s *Snapshot) Unmarshal(b []byte) error { return json.Unmarshal(b, s) } + +// loadedValue is the outcome of loading a single source. +type loadedValue struct { + key string + raw json.RawMessage + avail bool +} + +// observe runs every source loader, returning one entry per source. Loader +// errors are treated as Unavailable (stale-while-revalidate) rather than +// failing the whole observation. +func (c *SystemContext) observe() []loadedValue { + out := make([]loadedValue, len(c.sources)) + for i, s := range c.sources { + out[i] = loadedValue{key: s.key.String()} + raw, err := s.load() + if err != nil { + out[i].avail = false + continue + } + out[i].raw = raw + out[i].avail = true + } + return out +} diff --git a/internal/systemcontext/context_test.go b/internal/systemcontext/context_test.go new file mode 100644 index 00000000..f1ab2070 --- /dev/null +++ b/internal/systemcontext/context_test.go @@ -0,0 +1,300 @@ +package systemcontext + +import ( + "encoding/json" + "sync" + "testing" +) + +// statefulSource is a source whose value can be changed by the test. +type statefulSource struct { + mu sync.Mutex + value string + loadErr bool + removable bool +} + +func newStateful(value string) *statefulSource { + return &statefulSource{value: value} +} + +func (s *statefulSource) load() (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.loadErr { + return "", errLoad + } + return s.value, nil +} + +func (s *statefulSource) set(v string) { + s.mu.Lock() + s.value = v + s.mu.Unlock() +} + +var errLoad = &jsonError{"load failed"} + +type jsonError struct{ s string } + +func (e *jsonError) Error() string { return e.s } + +func strCodec() Codec[string] { + return JSONCodec(func(a, b string) bool { return a == b }) +} + +func makeSrc(s *statefulSource) Source[string] { + return Source[string]{ + Key: NewKey("test", "value"), + Codec: strCodec(), + Load: s.load, + Baseline: func(v string) string { return "BASE[" + v + "]" }, + Update: func(_, cur string) string { return "UPDATE[" + cur + "]" }, + } +} + +func TestInitializeRendersDeterministicBaseline(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + + base, snap, err := r.Initialize() + if err != nil { + t.Fatalf("Initialize: %v", err) + } + if base != "BASE[v1]" { + t.Fatalf("baseline = %q, want %q", base, "BASE[v1]") + } + if snap == nil || snap.Values["test/value"] == nil { + t.Fatalf("snapshot missing value") + } + if _, err := snap.Marshal(); err != nil { + t.Fatalf("snapshot marshal: %v", err) + } +} + +func TestReconcileUnchanged(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + _, snap, _ := r.Initialize() + + res := r.Reconcile(snap) + if res.Action != Unchanged { + t.Fatalf("action = %v, want Unchanged", res.Action) + } +} + +func TestReconcileUpdate(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + _, snap, _ := r.Initialize() + + s.set("v2") + res := r.Reconcile(snap) + if res.Action != Updated { + t.Fatalf("action = %v, want Updated", res.Action) + } + if res.Text != "UPDATE[v2]" { + t.Fatalf("text = %q, want %q", res.Text, "UPDATE[v2]") + } + if res.Snapshot == nil { + t.Fatal("advanced snapshot missing") + } + + // A subsequent reconcile with the advanced snapshot is unchanged. + again := r.Reconcile(res.Snapshot) + if again.Action != Unchanged { + t.Fatalf("second action = %v, want Unchanged", again.Action) + } +} + +func TestReconcileNewSourceEmitsBaselineOnce(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + // Start with an empty snapshot: the source is new. + res := r.Reconcile(NewSnapshot()) + if res.Action != Updated { + t.Fatalf("action = %v, want Updated", res.Action) + } + if res.Text != "BASE[v1]" { + t.Fatalf("text = %q, want %q", res.Text, "BASE[v1]") + } +} + +func TestUnavailableRetainsPriorValue(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + _, snap, _ := r.Initialize() + + // Mark the source unavailable; reconcile should retain the prior value. + s.loadErr = true + res := r.Reconcile(snap) + if res.Action != Unchanged { + t.Fatalf("action = %v, want Unchanged (stale-while-revalidate)", res.Action) + } +} + +func TestInitializeBlocksOnUnavailable(t *testing.T) { + s := newStateful("v1") + s.loadErr = true + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + _, _, err := r.Initialize() + if err == nil { + t.Fatal("expected InitializationBlocked") + } + if _, ok := err.(*InitializationBlocked); !ok { + t.Fatalf("err = %T, want *InitializationBlocked", err) + } +} + +func TestRemovableSourceRemovalRendersRemoval(t *testing.T) { + // A dynamic context where sources can appear/disappear requires a host that + // rebuilds the SystemContext. Here we simulate by using two sources and a + // snapshot that records both, then reconcile with a context missing one. + a := newStateful("a1") + b := newStateful("b1") + ctxAB := NewAll(sourceA(a), sourceB(b)) + rAB := NewReconciler(ctxAB) + _, snap, _ := rAB.Initialize() + + // Rebuild with only source A. Source B's key is still in the snapshot but + // absent from the new context. + ctxA := New(sourceA(a)) + rA := NewReconciler(ctxA) + res := rA.Reconcile(snap) + if res.Action != Replace { + // Composition change => replacement required. + t.Fatalf("action = %v, want Replace", res.Action) + } +} + +func sourceA(s *statefulSource) Source[string] { + return Source[string]{ + Key: NewKey("test", "a"), + Codec: strCodec(), + Load: s.load, + Baseline: func(v string) string { return "A[" + v + "]" }, + Update: func(_, cur string) string { return "A-UPD[" + cur + "]" }, + } +} + +func sourceB(s *statefulSource) Source[string] { + return Source[string]{ + Key: NewKey("test", "b"), + Codec: strCodec(), + Load: s.load, + Baseline: func(v string) string { return "B[" + v + "]" }, + Update: func(_, cur string) string { return "B-UPD[" + cur + "]" }, + } +} + +func TestEpochPrepareUpdateAndBaselineStable(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + base, snap, _ := r.Initialize() + ep := NewEpoch(base, snap) + origBase := ep.Baseline + + s.set("v2") + upd, replace, _, err := ep.Prepare(r) + if err != nil { + t.Fatalf("Prepare: %v", err) + } + if replace { + t.Fatal("unexpected replace") + } + if upd == nil || *upd != "UPDATE[v2]" { + t.Fatalf("update = %v, want UPDATE[v2]", upd) + } + // Baseline must remain unchanged across an update. + if ep.Baseline != origBase { + t.Fatalf("baseline changed on update") + } + + // Next prepare is unchanged. + upd2, replace2, _, err := ep.Prepare(r) + if err != nil || upd2 != nil || replace2 { + t.Fatalf("second prepare = (%v,%v,%v), want unchanged", upd2, replace2, err) + } +} + +func TestEpochReplaceFoldsFreshBaseline(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + base, snap, _ := r.Initialize() + ep := NewEpoch(base, snap) + + // A compaction-like transition replaces the baseline. + newBase, newSnap, err := r.Replace(ep.Snapshot) + if err != nil { + t.Fatalf("Replace: %v", err) + } + if newBase != "BASE[v1]" { + t.Fatalf("new baseline = %q", newBase) + } + ep.Baseline = newBase + ep.Snapshot = newSnap + if ep.Baseline != "BASE[v1]" { + t.Fatalf("epoch baseline not replaced") + } +} + +func TestDuplicateKeyPanics(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("expected panic on duplicate key") + } + }() + s := newStateful("v1") + NewAll(makeSrc(s), makeSrc(s)) +} + +func TestSnapshotMarshalRoundTrip(t *testing.T) { + s := newStateful("v1") + ctx := New(makeSrc(s)) + r := NewReconciler(ctx) + _, snap, _ := r.Initialize() + + b, err := snap.Marshal() + if err != nil { + t.Fatalf("marshal: %v", err) + } + var restored Snapshot + if err := restored.Unmarshal(b); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if string(restored.Values["test/value"]) == "" { + t.Fatal("round-trip lost value") + } + + // Reconciling against the restored snapshot should be unchanged. + res := r.Reconcile(&restored) + if res.Action != Unchanged { + t.Fatalf("action against restored snapshot = %v, want Unchanged", res.Action) + } +} + +func TestJSONEqualityIgnoresWhitespace(t *testing.T) { + // A codec with no Equal falls back to normalized JSON comparison. + c := Codec[json.RawMessage]{ + Encode: func(v json.RawMessage) (json.RawMessage, error) { return v, nil }, + Decode: func(b json.RawMessage) (json.RawMessage, error) { return b, nil }, + } + a := json.RawMessage(`{"x":1}`) + b := json.RawMessage(`{"x": 1}`) + if !equalJSON(c, a, b) { + t.Fatal("expected normalized JSON equality") + } + // Different values must compare unequal. + c2 := json.RawMessage(`{"x":2}`) + if equalJSON(c, a, c2) { + t.Fatal("expected inequality for different values") + } +} diff --git a/internal/systemcontext/doc.go b/internal/systemcontext/doc.go new file mode 100644 index 00000000..23f6ee8a --- /dev/null +++ b/internal/systemcontext/doc.go @@ -0,0 +1,5 @@ +// Package systemcontext implements a composable, incrementally-rendered model +// context runtime. +// +// See the package-level comment in context.go for the full design. +package systemcontext diff --git a/internal/systemcontext/epoch.go b/internal/systemcontext/epoch.go new file mode 100644 index 00000000..a9c7694c --- /dev/null +++ b/internal/systemcontext/epoch.go @@ -0,0 +1,49 @@ +package systemcontext + +// Epoch ties a durable baseline, its snapshot, and the sequence bookkeeping +// together. An Epoch starts at a fresh baseline and advances atomically as +// changed context is admitted at safe provider-turn boundaries. +// +// This mirrors opencode's Context Epoch: the baseline remains the immutable +// provider-cache prefix for the span of the epoch, and only delta updates are +// admitted as chronological messages. +type Epoch struct { + // Baseline is the exact, immutable system-prompt prefix for this epoch. + Baseline string + // Snapshot is the model-hidden comparison state that advances with updates. + Snapshot *Snapshot +} + +// NewEpoch constructs an epoch from an already-computed baseline and snapshot. +func NewEpoch(baseline string, snap *Snapshot) *Epoch { + if snap == nil { + snap = NewSnapshot() + } + return &Epoch{Baseline: baseline, Snapshot: snap} +} + +// Prepare admits context changes at a safe provider-turn boundary. +// +// It returns nil when nothing changed, an update message when one or more +// sources changed (advancing the epoch's snapshot), or (true, _, _) when the +// baseline must be replaced. +func (e *Epoch) Prepare(r *Reconciler) (update *string, replace bool, replaceBaseline string, err error) { + res := r.Reconcile(e.Snapshot) + switch res.Action { + case Unchanged: + return nil, false, "", nil + case Updated: + e.Snapshot = res.Snapshot + t := res.Text + return &t, false, "", nil + case Replace: + base, snap, err := r.Replace(e.Snapshot) + if err != nil { + return nil, true, "", err + } + e.Baseline = base + e.Snapshot = snap + return nil, true, base, nil + } + return nil, false, "", nil +} diff --git a/internal/systemcontext/reconciler.go b/internal/systemcontext/reconciler.go new file mode 100644 index 00000000..57f051da --- /dev/null +++ b/internal/systemcontext/reconciler.go @@ -0,0 +1,195 @@ +package systemcontext + +import ( + "encoding/json" + "sort" + "strings" +) + +// Action is the outcome of reconciling current context against a snapshot. +type Action int + +const ( + // Unchanged means no model-visible context change needs to be admitted. + Unchanged Action = iota + // Updated means at least one source changed; Text carries the single + // combined mid-conversation system message to emit. + Updated + // Replace means the baseline can no longer be reused and a fresh baseline + // must be rendered. + Replace +) + +// Result is the outcome of a reconcile operation. +type Result struct { + Action Action + Text string + // Snapshot is the advanced snapshot to persist when Action == Updated. + Snapshot *Snapshot +} + +// Reconciler compares loaded source values against a durable snapshot and +// produces incremental updates. It is the core of the incremental-context +// pattern: only changed sources render text, so an immutable baseline is +// preserved across turns. +type Reconciler struct { + ctx *SystemContext +} + +// NewReconciler wraps a composed SystemContext. +func NewReconciler(ctx *SystemContext) *Reconciler { + return &Reconciler{ctx: ctx} +} + +// LoadAll returns the current raw value of every source. It is used to build +// an initial baseline. +func (r *Reconciler) LoadAll() (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + entries := r.ctx.observe() + for _, e := range entries { + if !e.avail { + continue + } + out[e.key] = e.raw + } + return out, nil +} + +// RenderBaseline renders the full ordered baseline text for the given values. +func (r *Reconciler) RenderBaseline(values map[string]json.RawMessage) string { + var b strings.Builder + for _, e := range r.ordered() { + raw, ok := values[e.key.String()] + if !ok { + continue + } + if t := e.base(raw); t != "" { + if b.Len() > 0 { + b.WriteString("\n\n") + } + b.WriteString(t) + } + } + return b.String() +} + +// Initialize produces a fresh baseline and its snapshot. If any source is +// unavailable it returns an error so callers never persist an incomplete +// baseline. +func (r *Reconciler) Initialize() (baseline string, snap *Snapshot, err error) { + entries := r.ctx.observe() + snap = NewSnapshot() + for _, e := range entries { + if !e.avail { + return "", nil, &InitializationBlocked{Key: e.key} + } + snap.Values[e.key] = e.raw + } + return r.RenderBaseline(snap.Values), snap, nil +} + +// InitializationBlocked reports that an unavailable source blocked baseline +// initialization. +type InitializationBlocked struct{ Key string } + +func (e *InitializationBlocked) Error() string { + return "systemcontext: source " + e.Key + " unavailable during initialization" +} + +// Reconcile compares current context against the provided snapshot and returns +// exactly one action. +// +// - Unchanged: no changes. +// - Updated: Text is the single combined system message; Snapshot is the +// advanced snapshot to persist. +// - Replace: baseline can no longer be reused (a removable source vanished +// without a removal renderer, or an incompatible transition). +func (r *Reconciler) Reconcile(prev *Snapshot) *Result { + if prev == nil { + prev = NewSnapshot() + } + entries := r.ctx.observe() + + var parts []string + next := NewSnapshot() + changed := false + + for _, e := range entries { + src := r.source(e.key) + if src == nil { + // Source not registered in this context; treat as replacement to be + // safe rather than silently dropping. + return &Result{Action: Replace} + } + if !e.avail { + // Unavailable: retain prior effective value (stale-while-revalidate). + if prevRaw, ok := prev.Values[e.key]; ok { + next.Values[e.key] = prevRaw + } + continue + } + prevRaw, had := prev.Values[e.key] + if !had { + // New source: emit baseline once. + if t := src.base(e.raw); t != "" { + parts = append(parts, t) + } + changed = true + } else if !src.equal(prevRaw, e.raw) { + if t := src.upd(prevRaw, e.raw); t != "" { + parts = append(parts, t) + } + changed = true + } + next.Values[e.key] = e.raw + } + + // Detect removed sources: a previously-admitted key with no current value. + // Because sources form a fixed set, a vanished source means the composition + // changed and the baseline must be rebuilt. + for key := range prev.Values { + if _, present := next.Values[key]; present { + continue + } + return &Result{Action: Replace} + } + + if !changed { + return &Result{Action: Unchanged} + } + return &Result{Action: Updated, Text: strings.Join(parts, "\n\n"), Snapshot: next} +} + +// Replace renders a fresh baseline after a baseline-replacing transition (e.g. +// compaction). It reports replacement blocked while previously-admitted context +// is unavailable. +func (r *Reconciler) Replace(prev *Snapshot) (baseline string, snap *Snapshot, err error) { + if prev != nil { + for key := range prev.Values { + if r.source(key) == nil { + return "", nil, &InitializationBlocked{Key: key} + } + } + } + return r.Initialize() +} + +// source returns the typedSource for a stable key string. +func (r *Reconciler) source(key string) *typedSource { + for i := range r.ctx.sources { + if r.ctx.sources[i].key.String() == key { + return &r.ctx.sources[i] + } + } + return nil +} + +// ordered returns the sources sorted by stable key for deterministic rendering. +func (r *Reconciler) ordered() []typedSource { + out := make([]typedSource, len(r.ctx.sources)) + copy(out, r.ctx.sources) + sort.SliceStable(out, func(i, j int) bool { + return out[i].key.String() < out[j].key.String() + }) + return out +}