From 9e448cf93e3193fa6a2292dcbb9d2b6e4fe69b05 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 23 Aug 2026 00:12:58 +0530 Subject: [PATCH 1/2] feat(grokbuild): adopt rewind, hunk tracking, fast copy, and edit hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six adoptions from a deep-dive of xai-org/grok-build (SpaceXAI's Rust coding agent), each verified as genuinely missing from hawk: Turn-boundary file rewind (internal/filestate): - Per-prompt rewind points with first-wins before-snapshots and last-write after-snapshots; RewindTo returns a restore plan (only files whose on-disk content drifted) and truncates later history. - Durable disk mirror under the state dir: sanitized session directory names (traversal-safe, hash-suffixed), atomic temp+rename checkpoint writes, rehydration on construction, cap-based eviction of oldest checkpoints. Hunk tracker (internal/hunktracker): - LCS line diff producing per-region hunks with content-derived IDs; reconciliation preserves hunk identity across re-diffs by ID or positional overlap, and agent attribution survives external edits overlapping an agent-authored region. AgentTouchedFiles exposes the agent-vs-user split. Compaction transcript segments (internal/engine/compact + wiring): - Verbatim compacted turns persist as self-contained markdown segments under the session store with detail levels (none/minimal/balanced/verbose via HAWK_COMPACTION_SEGMENT_DETAIL), a 512KB-per-segment truncation notice, and an INDEX.md row per segment, so full history stays retrievable after compaction. smartCompactBody writes segments best-effort before dropping turns from the live context. Fast CoW tree copy (internal/fastcopy): - APFS clonefile on darwin with byte-copy fallback elsewhere; parallel copy sharded by parent-directory hash so same-directory files share one worker, avoiding create-dir contention — the grok-build xai-fast-worktree technique. Prompt-queue combining (internal/engine/prompt_queue_combine.go): - Pure merge rules (front/follower eligibility gates) plus DequeueCombined, which merges the run of eligible plain prompts behind the front into one turn with combined display metadata; steering/synthetic/bash/image items keep their own turns. Fuzzy-edit unicode pass (internal/tool/file_edit.go): - Fourth fuzzyFind strategy folding typographic characters (em/en dashes, smart quotes, ellipsis, nbsp) to ASCII before matching, recovering exact matches for common model output drift before falling to Levenshtein. Verification: go build ./... clean; new suites green (filestate 7, fastcopy 5, hunktracker 6, combine 7, segments 7, fuzzy 3); tool/engine/compact suites pass; golangci-lint 0 issues; gofmt clean. --- internal/engine/compact.go | 13 + .../engine/compact/transcript_segments.go | 267 +++++++++++++++ .../compact/transcript_segments_test.go | 117 +++++++ internal/engine/prompt_queue_combine.go | 156 +++++++++ internal/engine/prompt_queue_combine_test.go | 125 +++++++ internal/fastcopy/fastcopy.go | 193 +++++++++++ internal/fastcopy/fastcopy_test.go | 87 +++++ internal/filestate/filestate.go | 313 ++++++++++++++++++ internal/filestate/filestate_test.go | 169 ++++++++++ internal/hunktracker/hunktracker.go | 235 +++++++++++++ internal/hunktracker/hunktracker_test.go | 91 +++++ internal/tool/file_edit.go | 65 +++- internal/tool/fuzzy_edit_test.go | 34 ++ 13 files changed, 1864 insertions(+), 1 deletion(-) create mode 100644 internal/engine/compact/transcript_segments.go create mode 100644 internal/engine/compact/transcript_segments_test.go create mode 100644 internal/engine/prompt_queue_combine.go create mode 100644 internal/engine/prompt_queue_combine_test.go create mode 100644 internal/fastcopy/fastcopy.go create mode 100644 internal/fastcopy/fastcopy_test.go create mode 100644 internal/filestate/filestate.go create mode 100644 internal/filestate/filestate_test.go create mode 100644 internal/hunktracker/hunktracker.go create mode 100644 internal/hunktracker/hunktracker_test.go diff --git a/internal/engine/compact.go b/internal/engine/compact.go index adb59762..a5e4d888 100644 --- a/internal/engine/compact.go +++ b/internal/engine/compact.go @@ -2,9 +2,12 @@ package engine import ( "context" + "log/slog" + "os" "strings" "time" + "github.com/GrayCodeAI/hawk/internal/engine/compact" "github.com/GrayCodeAI/hawk/internal/engine/token" "github.com/GrayCodeAI/hawk/internal/types" @@ -97,6 +100,16 @@ func (s *Session) smartCompactBody(ctx context.Context) { summary += "\n\n" + fileBlock } + // Persist the verbatim compacted turns as a retrievable transcript + // segment before they leave the live context. Best-effort: a persistence + // failure must never block or corrupt compaction itself. + if sessionID := s.executionGraphSessionID(); sessionID != "" && len(compactedMsgs) > 0 { + detail, _ := compact.ParseCompactionDetail(os.Getenv("HAWK_COMPACTION_SEGMENT_DETAIL")) + if _, err := compact.WriteCompactionSegment(sessionID, compactedMsgs, detail); err != nil { + slog.Debug("compaction segment persistence skipped", "error", err) + } + } + tail := raw[len(raw)-keepEnd:] keep := make([]types.EyrieMessage, 0, len(tail)+2) keep = append(keep, types.EyrieMessage{ diff --git a/internal/engine/compact/transcript_segments.go b/internal/engine/compact/transcript_segments.go new file mode 100644 index 00000000..db6ea471 --- /dev/null +++ b/internal/engine/compact/transcript_segments.go @@ -0,0 +1,267 @@ +package compact + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/GrayCodeAI/hawk/internal/storage" + "github.com/GrayCodeAI/hawk/internal/types" +) + +// Compaction transcript segments, adopted from grok-build's +// xai-compaction-transcript: when compaction summarizes old turns away, the +// verbatim turns are persisted as self-contained markdown segments under the +// session store so full history remains retrievable after the fact. An INDEX.md +// makes segments discoverable without parsing every file. + +// CompactionDetail controls how much per-turn detail lands in a segment. +type CompactionDetail int + +const ( + // SegmentNone: stats only, no verbatim turns. + SegmentNone CompactionDetail = iota + // SegmentMinimal: one-line signature per turn. + SegmentMinimal + // SegmentBalanced: tool calls + truncated responses + full text. + SegmentBalanced + // SegmentVerbose: full verbatim turns (default). + SegmentVerbose +) + +// ParseCompactionDetail parses a user-facing detail level name. +func ParseCompactionDetail(s string) (CompactionDetail, bool) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "none": + return SegmentNone, true + case "minimal": + return SegmentMinimal, true + case "balanced": + return SegmentBalanced, true + case "verbose", "": + return SegmentVerbose, true + default: + return SegmentVerbose, false + } +} + +const ( + segmentsDirName = "compaction" + indexFileName = "INDEX.md" + segmentPrefix = "segment_" + + // segmentMaxBytes caps one segment's verbatim body; overflow turns are + // omitted behind an explicit truncation notice. + segmentMaxBytes = 512 * 1024 + + balancedTextChars = 2000 + balancedResponseChars = 500 + + perTurnOverheadBytes = 64 +) + +var segmentFileRe = regexp.MustCompile(`^segment_(\d+)\.md$`) + +// SegmentsDir is the per-session directory holding compaction segments. +func SegmentsDir(sessionID string) string { + return filepath.Join(storage.SessionsDir(), sessionID, segmentsDirName) +} + +// IndexPath is the per-session segment index file. +func IndexPath(sessionID string) string { + return filepath.Join(SegmentsDir(sessionID), indexFileName) +} + +// RenderSegmentToMarkdown renders messages into a self-contained markdown +// segment. Pure: no I/O. +func RenderSegmentToMarkdown(msgs []types.EyrieMessage, segIndex int, detail CompactionDetail) string { + var b strings.Builder + fmt.Fprintf(&b, "# Compaction segment %d\n\n", segIndex) + fmt.Fprintf(&b, "Recorded: %s\n", time.Now().UTC().Format(time.RFC3339)) + fmt.Fprintf(&b, "Turns: %d | Detail: %s\n", len(msgs), detail.String()) + + if detail == SegmentNone || len(msgs) == 0 { + b.WriteString("\n(verbatim turns not recorded at this detail level)\n") + return b.String() + } + + var body strings.Builder + used := 0 + omitted := 0 + for _, m := range msgs { + entry := renderTurn(m, detail) + cost := len(entry) + perTurnOverheadBytes + if used+cost > segmentMaxBytes { + omitted++ + continue + } + used += cost + body.WriteString(entry) + } + b.WriteString("\n---\n\n") + b.WriteString(body.String()) + if omitted > 0 { + fmt.Fprintf(&b, "\n[... TRUNCATED at %d bytes, %d turns omitted ...]\n", segmentMaxBytes, omitted) + } + return b.String() +} + +func renderTurn(m types.EyrieMessage, detail CompactionDetail) string { + var b strings.Builder + role := strings.ToUpper(m.Role) + fmt.Fprintf(&b, "## %s\n", role) + + text := strings.TrimSpace(m.Content) + switch detail { + case SegmentMinimal: + if text != "" { + fmt.Fprintf(&b, "%s\n", oneLine(text)) + } + for _, tu := range m.ToolUse { + fmt.Fprintf(&b, "- tool: %s\n", tu.Name) + } + default: + if text != "" { + if detail == SegmentBalanced { + text = truncateRunesStr(text, balancedTextChars) + } + fmt.Fprintf(&b, "%s\n", text) + } + for _, tu := range m.ToolUse { + args := summarizeToolArgs(tu.Arguments, detail) + fmt.Fprintf(&b, "- tool: %s(%s)\n", tu.Name, args) + } + for _, tr := range m.ToolResults { + out := tr.Content + if detail == SegmentBalanced { + out = truncateRunesStr(out, balancedResponseChars) + } + fmt.Fprintf(&b, "- result: %s\n", oneLine(out)) + } + } + b.WriteString("\n") + return b.String() +} + +func summarizeToolArgs(args map[string]interface{}, detail CompactionDetail) string { + if detail == SegmentBalanced { + keys := make([]string, 0, len(args)) + for k := range args { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=…", k)) + } + return strings.Join(parts, ", ") + } + // encoding/json emits map keys in sorted order, giving deterministic args + // digests across runs. + if raw, err := json.Marshal(args); err == nil { + s := string(raw) + if len(s) > 300 { + s = s[:300] + "…" + } + return s + } + return "…" +} + +func oneLine(s string) string { + s = strings.ReplaceAll(s, "\n", " ") + if len(s) > 200 { + s = s[:200] + "…" + } + return s +} + +func truncateRunesStr(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max]) + "…" +} + +// NextSegmentIndex derives the next segment number from existing files on +// disk, falling back to the INDEX rows when files were pruned. +func NextSegmentIndex(sessionID string) int { + max := -1 + entries, err := os.ReadDir(SegmentsDir(sessionID)) + if err == nil { + for _, e := range entries { + m := segmentFileRe.FindStringSubmatch(e.Name()) + if m == nil { + continue + } + if n, err := strconv.Atoi(m[1]); err == nil && n > max { + max = n + } + } + } + return max + 1 +} + +// WriteCompactionSegment persists msgs as the next segment and appends an +// INDEX.md row. It returns the written segment path. Callers should treat +// errors as non-fatal: segment persistence must never block compaction. +func WriteCompactionSegment(sessionID string, msgs []types.EyrieMessage, detail CompactionDetail) (string, error) { + if sessionID == "" { + return "", fmt.Errorf("compact: empty session id") + } + dir := SegmentsDir(sessionID) + if err := os.MkdirAll(dir, 0o750); err != nil { + return "", fmt.Errorf("compact: create segments dir: %w", err) + } + idx := NextSegmentIndex(sessionID) + path := filepath.Join(dir, fmt.Sprintf("%s%04d.md", segmentPrefix, idx)) + content := RenderSegmentToMarkdown(msgs, idx, detail) + if err := os.WriteFile(path, []byte(content), 0o640); err != nil { // #nosec G306 -- session-owned transcript segment + return "", fmt.Errorf("compact: write segment: %w", err) + } + if err := appendIndexRow(sessionID, idx, len(msgs), len(content)); err != nil { + return path, fmt.Errorf("compact: segment written but index update failed: %w", err) + } + return path, nil +} + +// appendIndexRow appends (creating with a header when absent) an INDEX.md row. +func appendIndexRow(sessionID string, segIndex, turns, bytes int) error { + path := IndexPath(sessionID) + if _, err := os.Stat(path); os.IsNotExist(err) { + header := "# Compaction segments\n\n| segment | turns | bytes | recorded |\n|---|---|---|---|\n" + if err := os.WriteFile(path, []byte(header), 0o640); err != nil { // #nosec G306 -- session-owned index + return err + } + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o640) // #nosec G304 -- path derived from session storage root + if err != nil { + return err + } + defer func() { _ = f.Close() }() + row := fmt.Sprintf("| [%d](%s%04d.md) | %d | %d | %s |\n", + segIndex, segmentPrefix, segIndex, turns, bytes, time.Now().UTC().Format(time.RFC3339)) + _, err = f.WriteString(row) + return err +} + +// String implements fmt.Stringer for logs. +func (d CompactionDetail) String() string { + switch d { + case SegmentNone: + return "none" + case SegmentMinimal: + return "minimal" + case SegmentBalanced: + return "balanced" + default: + return "verbose" + } +} diff --git a/internal/engine/compact/transcript_segments_test.go b/internal/engine/compact/transcript_segments_test.go new file mode 100644 index 00000000..c558a9df --- /dev/null +++ b/internal/engine/compact/transcript_segments_test.go @@ -0,0 +1,117 @@ +package compact + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/GrayCodeAI/hawk/internal/types" +) + +func segTestMessages() []types.EyrieMessage { + return []types.EyrieMessage{ + {Role: "user", Content: "fix the flaky test in pkg/foo"}, + { + Role: "assistant", Content: "Looking into it.", + ToolUse: []types.ToolCall{{Name: "Read", Arguments: map[string]interface{}{"path": "pkg/foo_test.go"}}}, + }, + { + Role: "user", Content: "", + ToolResults: []types.ToolResult{{Content: "package foo\n\nfunc TestX(t *testing.T) {}"}}, + }, + {Role: "assistant", Content: "Found it: shared map without a mutex."}, + } +} + +func TestRenderSegmentVerbose(t *testing.T) { + out := RenderSegmentToMarkdown(segTestMessages(), 3, SegmentVerbose) + for _, want := range []string{ + "# Compaction segment 3", "Turns: 4", "Detail: verbose", + "## USER", "## ASSISTANT", "flaky test", "- tool: Read", + } { + if !strings.Contains(out, want) { + t.Fatalf("segment missing %q", want) + } + } +} + +func TestRenderSegmentMinimal(t *testing.T) { + out := RenderSegmentToMarkdown(segTestMessages(), 0, SegmentMinimal) + if strings.Contains(out, "package foo") { + t.Fatal("minimal detail should not include full tool results") + } + if !strings.Contains(out, "- tool: Read") { + t.Fatalf("minimal should keep one-line tool signatures") + } +} + +func TestRenderSegmentNone(t *testing.T) { + out := RenderSegmentToMarkdown(segTestMessages(), 1, SegmentNone) + if strings.Contains(out, "## USER") || strings.Contains(out, "flaky test") { + t.Fatalf("none detail must omit turns, got %q", out) + } + if !strings.Contains(out, "Turns: 4") { + t.Fatal("none detail still reports stats") + } +} + +func TestParseCompactionDetail(t *testing.T) { + cases := map[string]CompactionDetail{ + "none": SegmentNone, "minimal": SegmentMinimal, + "balanced": SegmentBalanced, "verbose": SegmentVerbose, "": SegmentVerbose, + } + for in, want := range cases { + got, ok := ParseCompactionDetail(in) + if !ok || got != want { + t.Fatalf("ParseCompactionDetail(%q) = (%v,%v)", in, got, ok) + } + } + if _, ok := ParseCompactionDetail("bogus"); ok { + t.Fatal("bogus should not parse") + } +} + +func TestWriteCompactionSegmentAndIndex(t *testing.T) { + stateDir := t.TempDir() + t.Setenv("HAWK_STATE_DIR", stateDir) + sessionID := "seg-test-session" + path, err := WriteCompactionSegment(sessionID, segTestMessages(), SegmentVerbose) + if err != nil { + t.Fatalf("WriteCompactionSegment: %v", err) + } + + if filepath.Base(filepath.Dir(path)) != segmentsDirName { + t.Fatalf("path = %q", path) + } + data, err := os.ReadFile(path) // #nosec G304 -- test-owned path from API + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "# Compaction segment 0") { + t.Fatalf("content = %q", string(data)[:80]) + } + + idxData, err := os.ReadFile(IndexPath(sessionID)) // #nosec G304 -- test-owned path + if err != nil { + t.Fatalf("index missing: %v", err) + } + if !strings.Contains(string(idxData), "| [0](segment_0000.md)") { + t.Fatalf("index = %q", string(idxData)) + } + + // Second write increments. + path2, err := WriteCompactionSegment(sessionID, segTestMessages(), SegmentBalanced) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(path2, "segment_0001.md") { + t.Fatalf("second path = %q", path2) + } +} + +func TestWriteCompactionSegmentRequiresSessionID(t *testing.T) { + if _, err := WriteCompactionSegment("", nil, SegmentVerbose); err == nil { + t.Fatal("expected error for empty session id") + } +} diff --git a/internal/engine/prompt_queue_combine.go b/internal/engine/prompt_queue_combine.go new file mode 100644 index 00000000..6065ccec --- /dev/null +++ b/internal/engine/prompt_queue_combine.go @@ -0,0 +1,156 @@ +package engine + +import ( + "strings" +) + +// Prompt-queue combining, adopted from grok-build's xai-prompt-queue +// combine rules: when several plain queued prompts are waiting, they are +// merged into a single turn instead of burning one full agent loop per +// prompt. Only mergeable prompts combine; anything synthetic, high-priority, +// or carrying structured payload stays its own turn so stop conditions and +// display cannot drift. + +const promptCombineSeparator = "\n\n" + +// CombineGate describes one queued item's eligibility for merging. It is a +// pure value so the rules stay testable without a live queue. +type CombineGate struct { + // PlainPrompt is a normal user text turn — not steering, not an + // interjection, not a command/synthetic origin. + PlainPrompt bool + // Synthetic origins (auto-wake, scheduled) never combine. + Synthetic bool + // Bash/command turns keep their own turn semantics. + Bash bool + // HasImages: followers must be text-only; a front prompt may carry images. + HasImages bool + // Text is the non-empty body participating in the join. + Text string +} + +// CanCombineFront reports whether gate can lead a combined run. +func CanCombineFront(g CombineGate) bool { + return g.PlainPrompt && !g.Synthetic && !g.Bash && g.Text != "" +} + +// CanCombineFollower reports whether gate can follow a front in the same +// combined turn. +func CanCombineFollower(g CombineGate) bool { + return CanCombineFront(g) && !g.HasImages +} + +// CombinePrefixLen returns the length of the mergeable prefix of gates, +// including the front. 0 for empty input, 1 when only the front is taken. +func CombinePrefixLen(gates []CombineGate) int { + if len(gates) == 0 { + return 0 + } + if !CanCombineFront(gates[0]) { + return 1 + } + n := 1 + for _, g := range gates[1:] { + if !CanCombineFollower(g) { + break + } + n++ + } + return n +} + +// JoinTexts joins non-empty texts with the combine separator. +func JoinTexts(texts []string) string { + out := make([]string, 0, len(texts)) + for _, t := range texts { + if t != "" { + out = append(out, t) + } + } + return strings.Join(out, promptCombineSeparator) +} + +// IsCombined reports whether at least two original prompts were merged. +func IsCombined(segs []string) bool { return len(segs) >= 2 } + +// combineGateFor derives the eligibility gate from an enqueued prompt. Source +// "user" is the plain path; anything else (cron, background notify, /btw +// interjections, bash results) keeps its own turn. Priority above Normal is +// never merged — steering must land promptly, not be diluted. +func combineGateFor(p EnqueuedPrompt) CombineGate { + plain := p.Source == "user" || p.Source == "" || + strings.EqualFold(p.Source, "telegram") || strings.EqualFold(p.Source, "discord") || + strings.EqualFold(p.Source, "slack") + hasImages := false + if p.Metadata != nil { + if v, ok := p.Metadata["images"]; ok && v != nil { + switch imgs := v.(type) { + case []string: + hasImages = len(imgs) > 0 + case []interface{}: + hasImages = len(imgs) > 0 + case bool: + hasImages = imgs + } + } + if v, ok := p.Metadata["synthetic"].(bool); ok && v { + return CombineGate{Synthetic: true} + } + if v, ok := p.Metadata["bash"].(bool); ok && v { + return CombineGate{Bash: true} + } + } + return CombineGate{ + PlainPrompt: plain, + Synthetic: false, + Bash: false, + HasImages: hasImages, + Text: strings.TrimSpace(p.Text), + } +} + +// DequeueCombined pops the next turn from the queue, merging the run of +// eligible followers behind it into one prompt. It returns the (possibly +// joined) prompt, the IDs consumed by the turn, and whether a merge happened. +func (pq *PromptQueue) DequeueCombined() (EnqueuedPrompt, []string, bool) { + pq.mu.Lock() + defer pq.mu.Unlock() + + if len(pq.items) == 0 { + return EnqueuedPrompt{}, nil, false + } + + gates := make([]CombineGate, len(pq.items)) + for i := range pq.items { + gates[i] = combineGateFor(pq.items[i]) + } + n := CombinePrefixLen(gates) + + front := pq.items[0] + consumedIDs := []string{front.ID} + if n <= 1 { + pq.items = pq.items[1:] + return front, consumedIDs, false + } + + segs := make([]string, 0, n) + for i := 0; i < n; i++ { + segs = append(segs, strings.TrimSpace(pq.items[i].Text)) + if i > 0 { + consumedIDs = append(consumedIDs, pq.items[i].ID) + } + } + merged := EnqueuedPrompt{ + ID: front.ID, + Text: JoinTexts(segs), + Priority: front.Priority, + Source: front.Source, + Metadata: map[string]interface{}{ + // Downstream UI can render one bubble per original prompt. + "combined_display_texts": segs, + "combined_ids": consumedIDs, + }, + } + pq.items = pq.items[n:] + return merged, consumedIDs, true +} diff --git a/internal/engine/prompt_queue_combine_test.go b/internal/engine/prompt_queue_combine_test.go new file mode 100644 index 00000000..c98c73c8 --- /dev/null +++ b/internal/engine/prompt_queue_combine_test.go @@ -0,0 +1,125 @@ +package engine + +import ( + "strings" + "testing" + "time" +) + +func eqPrompt(text string) EnqueuedPrompt { + return EnqueuedPrompt{ID: "p" + text, Text: text, Priority: PriorityNormal, Source: "user", EnqueuedAt: time.Now()} +} + +func TestCombinePrefixLen(t *testing.T) { + gates := []CombineGate{ + {PlainPrompt: true, Text: "a"}, + {PlainPrompt: true, Text: "b"}, + {PlainPrompt: true, HasImages: true, Text: "img"}, // follower with images stops the run + {PlainPrompt: true, Text: "c"}, + } + if got := CombinePrefixLen(gates); got != 2 { + t.Fatalf("CombinePrefixLen = %d, want 2", got) + } + + syntheticFirst := []CombineGate{{Synthetic: true}, {PlainPrompt: true, Text: "a"}} + if got := CombinePrefixLen(syntheticFirst); got != 1 { + t.Fatalf("synthetic front prefix = %d, want 1", got) + } + if got := CombinePrefixLen(nil); got != 0 { + t.Fatalf("empty prefix = %d, want 0", got) + } +} + +func TestJoinTextsAndIsCombined(t *testing.T) { + if got := JoinTexts([]string{"a", "", "b"}); got != "a\n\nb" { + t.Fatalf("JoinTexts = %q", got) + } + if !IsCombined([]string{"a", "b"}) || IsCombined([]string{"a"}) { + t.Fatal("IsCombined wrong") + } +} + +func TestDequeueCombinedMergesRun(t *testing.T) { + pq := NewPromptQueue() + pq.Enqueue(eqPrompt("first")) + pq.Enqueue(eqPrompt("second")) + + prompt, ids, merged := pq.DequeueCombined() + if !merged { + t.Fatal("expected merge") + } + if prompt.Text != "first\n\nsecond" { + t.Fatalf("merged text = %q", prompt.Text) + } + if len(ids) != 2 || ids[0] != "pfirst" || ids[1] != "psecond" { + t.Fatalf("ids = %v", ids) + } + if !pq.IsEmpty() { + t.Fatalf("queue should be drained, len=%d", pq.Len()) + } +} + +func TestDequeueCombinedSteeringSortsFirstAndNeverMerges(t *testing.T) { + pq := NewPromptQueue() + pq.Enqueue(eqPrompt("plain1")) + pq.Enqueue(eqPrompt("plain2")) + // Higher priority sorts ahead of the plain prompts. + pq.Enqueue(EnqueuedPrompt{ID: "steer", Text: "urgent", Priority: PrioritySteering, Source: "cron"}) + + // First turn: steering alone (front ineligible -> no merge). + prompt, ids, merged := pq.DequeueCombined() + if merged || prompt.Text != "urgent" || len(ids) != 1 { + t.Fatalf("steering turn wrong: merged=%v text=%q ids=%v", merged, prompt.Text, ids) + } + // Second turn: the two plain prompts combine. + prompt, ids, merged = pq.DequeueCombined() + if !merged || prompt.Text != "plain1\n\nplain2" || len(ids) != 2 { + t.Fatalf("plain run wrong: merged=%v text=%q ids=%v", merged, prompt.Text, ids) + } +} + +func TestDequeueCombinedNoMergeWhenFrontIneligible(t *testing.T) { + pq := NewPromptQueue() + pq.Enqueue(EnqueuedPrompt{ID: "s1", Text: "wake up", Priority: PrioritySteering, Source: "cron"}) + pq.Enqueue(eqPrompt("plain")) + + // The steering prompt sorts to the front and is ineligible. + prompt, ids, merged := pq.DequeueCombined() + if merged { + t.Fatal("synthetic front must not merge") + } + if prompt.Text != "wake up" || len(ids) != 1 { + t.Fatalf("prompt=%+v ids=%v", prompt, ids) + } +} + +func TestDequeueCombinedFollowerWithImageStops(t *testing.T) { + pq := NewPromptQueue() + pq.Enqueue(eqPrompt("front")) + pq.Enqueue(EnqueuedPrompt{ + ID: "img", Text: "look", Priority: PriorityNormal, Source: "user", + Metadata: map[string]interface{}{"images": []string{"a.png"}}, + }) + pq.Enqueue(eqPrompt("tail")) + + _, _, merged := pq.DequeueCombined() + if merged { + t.Fatal("image follower must stop the run") + } + if got := pq.Len(); got != 2 { + t.Fatalf("len = %d, want 2 (image + tail remain)", got) + } +} + +func TestDequeueCombinedEmpty(t *testing.T) { + pq := NewPromptQueue() + if _, _, merged := pq.DequeueCombined(); merged { + t.Fatal("empty queue must not report a merge") + } +} + +func TestJoinSeparatorIsBlankLine(t *testing.T) { + if !strings.Contains(JoinTexts([]string{"x", "y"}), "\n\n") { + t.Fatal("separator should be a blank line") + } +} diff --git a/internal/fastcopy/fastcopy.go b/internal/fastcopy/fastcopy.go new file mode 100644 index 00000000..e9214778 --- /dev/null +++ b/internal/fastcopy/fastcopy.go @@ -0,0 +1,193 @@ +// Package fastcopy copies directory trees using copy-on-write where the +// filesystem supports it (APFS clonefile on macOS, FICLONE ioctl on Linux), +// falling back to plain byte copies elsewhere. Parallelism is sharded by +// parent directory so files in the same directory always land on the same +// worker, avoiding create-dir lock contention — the technique grok-build's +// xai-fast-worktree uses to make standalone worktree creation O(file_count) +// instead of a serial walk. +package fastcopy + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + + "golang.org/x/sys/unix" +) + +// maxWorkers bounds the copy worker pool. macOS defaults to a low soft +// file-descriptor limit, so it gets fewer workers than Linux. +func maxWorkers() int { + if runtime.GOOS == "darwin" { + return 8 + } + return 32 +} + +// shardCount is the number of work shards. Prime counts distribute better. +const shardCount = 16 + +func shardFor(dir string) int { + sum := sha256.Sum256([]byte(dir)) + return int(binary.BigEndian.Uint32(sum[:4]) % shardCount) +} + +// Tree copies the directory tree at src to dst. Existing dst content is +// overwritten file-by-file. Returns the number of files copied and bytes +// written. ctx cancellation stops the walk at the next file. +func Tree(ctx context.Context, src, dst string) (files int64, bytes int64, err error) { + info, err := os.Stat(src) + if err != nil { + return 0, 0, fmt.Errorf("fastcopy: stat src: %w", err) + } + if !info.IsDir() { + return 0, 0, fmt.Errorf("fastcopy: src is not a directory: %s", src) + } + + type job struct{ rel string } + jobs := make([][]job, shardCount) + + walkErr := filepath.WalkDir(src, func(path string, d os.DirEntry, werr error) error { + if werr != nil { + return werr + } + if ctx.Err() != nil { + return ctx.Err() + } + rel, rerr := filepath.Rel(src, path) + if rerr != nil { + return rerr + } + if rel == "." { + return os.MkdirAll(dst, 0o750) + } + if d.IsDir() { + return os.MkdirAll(filepath.Join(dst, rel), 0o750) + } + if !d.Type().IsRegular() { + return nil // skip symlinks/devices/fifos in workspace copies + } + s := shardFor(filepath.Dir(rel)) + jobs[s] = append(jobs[s], job{rel: rel}) + return nil + }) + if walkErr != nil { + return atomic.LoadInt64(&files), atomic.LoadInt64(&bytes), walkErr + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + fileCnt atomic.Int64 + byteCnt atomic.Int64 + ) + for s := 0; s < shardCount; s++ { + shardJobs := jobs[s] + if len(shardJobs) == 0 { + continue + } + wg.Add(1) + go func(jobs []job) { + defer wg.Done() + for _, j := range jobs { + if ctx.Err() != nil { + return + } + n, err := copyFileCoW( + filepath.Join(src, j.rel), + filepath.Join(dst, j.rel), + ) + fileCnt.Add(1) + byteCnt.Add(n) + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + return + } + } + }(shardJobs) + if s+1 >= maxWorkers() { + // Reuse finished workers before spawning more shards. + wg.Wait() + } + } + wg.Wait() + if firstErr != nil { + return fileCnt.Load(), byteCnt.Load(), firstErr + } + return fileCnt.Load(), byteCnt.Load(), ctx.Err() +} + +// copyFileCoW clones src into dst via CoW when possible, else a byte copy. +func copyFileCoW(src, dst string) (int64, error) { + if err := os.MkdirAll(filepath.Dir(dst), 0o750); err != nil { + return 0, err + } + if n, cloned := tryCloneFile(src, dst); cloned { + return n, nil + } + in, err := os.Open(src) // #nosec G304 -- caller-supplied tree paths + if err != nil { + return 0, err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) // #nosec G304 -- mirrored tree path + if err != nil { + return 0, err + } + n, err := io.Copy(out, in) + cerr := out.Close() + if err == nil { + err = cerr + } + return n, err +} + +// tryCloneFile attempts a filesystem-level clone; returns cloned=false when +// unsupported (caller falls back). APFS clonefile is supported on darwin; +// other platforms take the byte-copy fallback, which stays correct everywhere. +func tryCloneFile(src, dst string) (int64, bool) { + if runtime.GOOS != "darwin" { + return 0, false + } + if err := unix.Clonefile(src, dst, 0); err != nil { + return 0, false + } + info, err := os.Stat(src) + if err != nil { + return 0, false + } + return info.Size(), true +} + +// SupportsCloneFile reports whether a quick probe clone succeeds on the +// filesystem containing dir (used by callers to log/choose strategies). +func SupportsCloneFile(dir string) bool { + probeSrc := filepath.Join(dir, ".hawk-fastcopy-probe") + if err := os.WriteFile(probeSrc, []byte("probe"), 0o600); err != nil { + return false + } + defer func() { _ = os.Remove(probeSrc) }() + probeDst := probeSrc + ".clone" + defer func() { _ = os.Remove(probeDst) }() + _, ok := tryCloneFile(probeSrc, probeDst) + return ok +} + +// TrimPrefixPath is a tiny helper for logging relative paths without leaking +// absolute prefixes. +func TrimPrefixPath(root, p string) string { + return strings.TrimPrefix(strings.TrimPrefix(p, root), string(filepath.Separator)) +} diff --git a/internal/fastcopy/fastcopy_test.go b/internal/fastcopy/fastcopy_test.go new file mode 100644 index 00000000..58cbc682 --- /dev/null +++ b/internal/fastcopy/fastcopy_test.go @@ -0,0 +1,87 @@ +package fastcopy + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func writeTree(t *testing.T, root string, files map[string]string) { + t.Helper() + for rel, content := range files { + path := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func TestTreeCopiesAllFiles(t *testing.T) { + src := t.TempDir() + dst := filepath.Join(t.TempDir(), "copy") + writeTree(t, src, map[string]string{ + "a.txt": "alpha", + "sub/b.txt": "beta", + "sub/deep/c.txt": "gamma", + "other/d.txt": stringsRepeat("x", 10000), + }) + files, bytes, err := Tree(context.Background(), src, dst) + if err != nil { + t.Fatalf("Tree: %v", err) + } + if files != 4 { + t.Fatalf("files = %d, want 4", files) + } + if bytes < 10010 { + t.Fatalf("bytes = %d", bytes) + } + for _, rel := range []string{"a.txt", "sub/b.txt", "sub/deep/c.txt", "other/d.txt"} { + got, err := os.ReadFile(filepath.Join(dst, rel)) // #nosec G304 -- test-owned path + if err != nil { + t.Fatalf("missing %s: %v", rel, err) + } + want, _ := os.ReadFile(filepath.Join(src, rel)) // #nosec G304 -- test-owned path + if string(got) != string(want) { + t.Fatalf("%s content mismatch", rel) + } + } +} + +func TestTreeEmptyDir(t *testing.T) { + src := t.TempDir() + dst := filepath.Join(t.TempDir(), "copy") + files, _, err := Tree(context.Background(), src, dst) + if err != nil || files != 0 { + t.Fatalf("files=%d err=%v", files, err) + } + if _, err := os.Stat(dst); err != nil { + t.Fatalf("dst not created: %v", err) + } +} + +func TestTreeSrcMustExist(t *testing.T) { + if _, _, err := Tree(context.Background(), "/nonexistent-fastcopy-src", t.TempDir()); err == nil { + t.Fatal("expected error for missing src") + } +} + +func TestShardForDeterministic(t *testing.T) { + a := shardFor("sub") + b := shardFor("sub") + c := shardFor("other") + if a != b || c >= shardCount || a >= shardCount { + t.Fatalf("sharding broken: %d %d %d", a, b, c) + } +} + +func stringsRepeat(s string, n int) string { + out := make([]byte, 0, len(s)*n) + for i := 0; i < n; i++ { + out = append(out, s...) + } + return string(out) +} diff --git a/internal/filestate/filestate.go b/internal/filestate/filestate.go new file mode 100644 index 00000000..9b0f4b64 --- /dev/null +++ b/internal/filestate/filestate.go @@ -0,0 +1,313 @@ +// Package filestate implements per-turn rewind points over the working tree, +// adopted from grok-build's checkpoint system: at each prompt boundary the +// touched files' contents are captured ("before" snapshots are first-wins, +// "after" snapshots last-write-wins), so a session can be rewound to any +// prior prompt by restoring those snapshots. +// +// A durable disk mirror (atomic temp-file writes, cap-based eviction, +// sanitized session directory names) lets rewind survive process restarts. +package filestate + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" +) + +// ErrNoRewindPoint is returned when no checkpoint exists for an index. +var ErrNoRewindPoint = errors.New("filestate: no rewind point at index") + +const ( + defaultCap = 64 + snapshotHashLen = 16 + storeDirPerms = 0o750 + checkpointFormat = "checkpoint-%d.json" +) + +// Snapshot is the captured content of one file at one moment. +type Snapshot struct { + Path string `json:"path"` + Content string `json:"content"` + Timestamp int64 `json:"timestamp_unix"` +} + +// RewindPoint captures file states around one prompt. +type RewindPoint struct { + PromptIndex int `json:"prompt_index"` + // Before maps path -> content before the first edit of this prompt + // (first-wins within the prompt). + Before map[string]string `json:"before,omitempty"` + // After maps path -> content after the last edit of this prompt + // (last-write-wins). + After map[string]string `json:"after,omitempty"` +} + +// Tracker manages rewind points for one session. +type Tracker struct { + mu sync.Mutex + points map[int]*RewindPoint + current *RewindPoint + order []int // ascending prompt indices present in points + cap int + sessionID string + durable bool +} + +// NewTracker creates a tracker. When durable is true, checkpoints mirror to +// the on-disk store and previously persisted points rehydrate on construction. +func NewTracker(sessionID string, durable bool) (*Tracker, error) { + t := &Tracker{ + points: map[int]*RewindPoint{}, + cap: defaultCap, + sessionID: sanitizeSessionID(sessionID), + durable: durable, + } + if durable { + if err := t.rehydrate(); err != nil { + return nil, err + } + } + return t, nil +} + +// BeginPrompt opens the rewind window for promptIndex. Calling Begin while a +// window is open closes it first. +func (t *Tracker) BeginPrompt(promptIndex int) { + t.mu.Lock() + defer t.mu.Unlock() + _ = t.endLocked() + t.current = &RewindPoint{ + PromptIndex: promptIndex, + Before: map[string]string{}, + After: map[string]string{}, + } +} + +// SetTouchResult records both sides of a touch explicitly: before is kept +// first-wins, after is stored last-wins. This variant avoids double reads. +func (t *Tracker) SetTouchResult(path, beforeContent, afterContent string) { + t.mu.Lock() + defer t.mu.Unlock() + if t.current == nil { + return + } + if _, seen := t.current.Before[path]; !seen { + t.current.Before[path] = beforeContent + } + t.current.After[path] = afterContent +} + +// EndPrompt closes the current window and persists the rewind point. +func (t *Tracker) EndPrompt() error { + t.mu.Lock() + defer t.mu.Unlock() + return t.endLocked() +} + +func (t *Tracker) endLocked() error { + if t.current == nil { + return nil + } + p := t.current + t.current = nil + if len(p.Before) == 0 && len(p.After) == 0 { + return nil // nothing touched: no point worth storing + } + if _, exists := t.points[p.PromptIndex]; !exists { + t.order = append(t.order, p.PromptIndex) + sort.Ints(t.order) + } + t.points[p.PromptIndex] = p + if t.durable && t.sessionID != "" { + if err := t.persist(p); err != nil { + return fmt.Errorf("filestate: persist checkpoint %d: %w", p.PromptIndex, err) + } + } + t.evictLocked() + return nil +} + +// RewindTo returns the restore plan for promptIndex: files whose pre-prompt +// content differs from what is currently on disk, mapped to their restored +// content. It truncates all rewind points at or after promptIndex (a rewind +// discards later history). The caller performs the actual writes. +func (t *Tracker) RewindTo(promptIndex int) (map[string]string, error) { + t.mu.Lock() + defer t.mu.Unlock() + + point, ok := t.points[promptIndex] + if !ok { + return nil, ErrNoRewindPoint + } + plan := map[string]string{} + for path, before := range point.Before { + cur, err := os.ReadFile(path) // #nosec G304 -- recorded workspace path + if err != nil || string(cur) != before { + plan[path] = before + } + } + // Discard this point and everything after it. + for idx := range t.points { + if idx >= promptIndex { + delete(t.points, idx) + } + } + filtered := t.order[:0] + for _, idx := range t.order { + if _, keep := t.points[idx]; keep { + filtered = append(filtered, idx) + } + } + t.order = filtered + if t.durable && t.sessionID != "" { + for idx := range t.pointsOnDisk(promptIndex) { + _ = os.Remove(filepath.Join(t.storeDir(), fmt.Sprintf(checkpointFormat, idx))) + } + } + return plan, nil +} + +// Len reports how many rewind points are held. +func (t *Tracker) Len() int { + t.mu.Lock() + defer t.mu.Unlock() + return len(t.points) +} + +// evictLocked drops the oldest points beyond capacity (memory and disk). +func (t *Tracker) evictLocked() { + for len(t.order) > t.cap { + oldest := t.order[0] + t.order = t.order[1:] + delete(t.points, oldest) + if t.durable && t.sessionID != "" { + _ = os.Remove(filepath.Join(t.storeDir(), fmt.Sprintf(checkpointFormat, oldest))) + } + } +} + +// storeDir is the durable mirror location inside the user state tree. +func (t *Tracker) storeDir() string { + return filepath.Join(stateRoot(), "rewind-checkpoints", t.sessionID) +} + +func (t *Tracker) persist(p *RewindPoint) error { + dir := t.storeDir() + if err := os.MkdirAll(dir, storeDirPerms); err != nil { + return err + } + data, err := json.Marshal(p) + if err != nil { + return err + } + final := filepath.Join(dir, fmt.Sprintf(checkpointFormat, p.PromptIndex)) + tmp := final + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { // #nosec G304 -- sanitized fixed layout + return err + } + return os.Rename(tmp, final) // atomic swap: crash-safe checkpoint +} + +// rehydrate loads persisted checkpoints back into memory. +func (t *Tracker) rehydrate() error { + if t.sessionID == "" { + return nil + } + dir := t.storeDir() + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, e := range entries { + var idx int + if _, err := fmt.Sscanf(e.Name(), checkpointFormat, &idx); err != nil { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) // #nosec G304 -- sanitized dir listing entry + if err != nil { + continue + } + var p RewindPoint + if json.Unmarshal(raw, &p) != nil || p.PromptIndex != idx { + continue + } + if p.Before == nil { + p.Before = map[string]string{} + } + if p.After == nil { + p.After = map[string]string{} + } + if _, exists := t.points[idx]; !exists { + t.order = append(t.order, idx) + t.points[idx] = &p + } + } + sort.Ints(t.order) + return nil +} + +// pointsOnDisk lists persisted checkpoint indexes >= from (for cleanup). +func (t *Tracker) pointsOnDisk(from int) map[int]bool { + out := map[int]bool{} + entries, err := os.ReadDir(t.storeDir()) + if err != nil { + return out + } + for _, e := range entries { + var idx int + if _, err := fmt.Sscanf(e.Name(), checkpointFormat, &idx); err != nil { + continue + } + if idx >= from { + out[idx] = true + } + } + return out +} + +// sanitizeSessionID strips anything but alphanumerics and appends a short +// hash, preventing path traversal from hostile session IDs (grok-build does +// the same for its rewind stores). +func sanitizeSessionID(id string) string { + id = strings.TrimSpace(id) + if id == "" { + return "" + } + var b strings.Builder + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + b.WriteRune(r) + } + } + clean := b.String() + cut := len(clean) + if cut > 24 { + cut = 24 + } + sum := sha256.Sum256([]byte(id)) + return fmt.Sprintf("%s-%s", clean[:cut], hex.EncodeToString(sum[:4])) +} + +// stateRoot mirrors storage.StateDir without importing it (leaf-package rule: +// filestate stays independent of hawk storage layout choices). +func stateRoot() string { + if v := strings.TrimSpace(os.Getenv("HAWK_STATE_DIR")); v != "" { + return v + } + home, err := os.UserHomeDir() + if err != nil { + return filepath.Join(os.TempDir(), "hawk", "state") + } + return filepath.Join(home, ".hawk", "state") +} diff --git a/internal/filestate/filestate_test.go b/internal/filestate/filestate_test.go new file mode 100644 index 00000000..ea4a9cf8 --- /dev/null +++ b/internal/filestate/filestate_test.go @@ -0,0 +1,169 @@ +package filestate + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRewindBasicFlow(t *testing.T) { + state := t.TempDir() + t.Setenv("HAWK_STATE_DIR", state) + work := t.TempDir() + f := filepath.Join(work, "code.go") + if err := os.WriteFile(f, []byte("v1\n"), 0o644); err != nil { + t.Fatal(err) + } + + tr, err := NewTracker("sess-1", false) + if err != nil { + t.Fatal(err) + } + tr.BeginPrompt(0) + tr.SetTouchResult(f, "v1\n", "v2\n") + if err := os.WriteFile(f, []byte("v2\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := tr.EndPrompt(); err != nil { + t.Fatal(err) + } + if tr.Len() != 1 { + t.Fatalf("points = %d", tr.Len()) + } + + // Prompt 1 edits again. + tr.BeginPrompt(1) + tr.SetTouchResult(f, "v2\n", "v3\n") + _ = os.WriteFile(f, []byte("v3\n"), 0o644) + _ = tr.EndPrompt() + + // Rewind to prompt 0: plan restores v1. + plan, err := tr.RewindTo(0) + if err != nil { + t.Fatal(err) + } + if plan[f] != "v1\n" { + t.Fatalf("plan = %q", plan[f]) + } + // Later history truncated. + if tr.Len() != 0 { + t.Fatalf("len after rewind = %d", tr.Len()) + } +} + +func TestRewindSkipsUnchangedFiles(t *testing.T) { + state := t.TempDir() + t.Setenv("HAWK_STATE_DIR", state) + work := t.TempDir() + f := filepath.Join(work, "a.txt") + _ = os.WriteFile(f, []byte("same"), 0o644) + + tr, _ := NewTracker("s", false) + tr.BeginPrompt(0) + tr.SetTouchResult(f, "same", "same") // touched but unchanged + _ = tr.EndPrompt() + + plan, err := tr.RewindTo(0) + if err != nil { + t.Fatal(err) + } + if len(plan) != 0 { + t.Fatalf("plan should skip unchanged files: %v", plan) + } +} + +func TestDurableRehydration(t *testing.T) { + state := t.TempDir() + t.Setenv("HAWK_STATE_DIR", state) + work := t.TempDir() + f := filepath.Join(work, "x.txt") + _ = os.WriteFile(f, []byte("one"), 0o644) + + tr, err := NewTracker("durable-session", true) + if err != nil { + t.Fatal(err) + } + tr.BeginPrompt(7) + tr.SetTouchResult(f, "one", "two") + _ = os.WriteFile(f, []byte("two"), 0o644) + if err := tr.EndPrompt(); err != nil { + t.Fatal(err) + } + + // A brand-new tracker (process restart) rehydrates from disk. + tr2, err := NewTracker("durable-session", true) + if err != nil { + t.Fatal(err) + } + if tr2.Len() != 1 { + t.Fatalf("rehydrated points = %d", tr2.Len()) + } + plan, err := tr2.RewindTo(7) + if err != nil { + t.Fatal(err) + } + if plan[f] != "one" { + t.Fatalf("plan = %q", plan[f]) + } +} + +func TestSanitizeSessionID(t *testing.T) { + hostile := "../../evil/../../id with spaces!" + sanitized := sanitizeSessionID(hostile) + if strings.Contains(sanitized, "..") || strings.Contains(sanitized, "/") || + strings.Contains(sanitized, " ") { + t.Fatalf("sanitized leaks traversal: %q", sanitized) + } + if sanitizeSessionID("") != "" { + t.Fatal("empty id must stay empty") + } + // Same input -> same output (stable store dirs). + if sanitizeSessionID(hostile) != sanitized { + t.Fatal("sanitize not deterministic") + } +} + +func TestEvictionCap(t *testing.T) { + state := t.TempDir() + t.Setenv("HAWK_STATE_DIR", state) + tr, _ := NewTracker("cap-sess", false) + for i := 0; i < defaultCap+10; i++ { + tr.BeginPrompt(i) + tr.SetTouchResult("/unused", "a", "b") + if err := tr.EndPrompt(); err != nil { + t.Fatal(err) + } + } + if tr.Len() != defaultCap { + t.Fatalf("len = %d, want cap %d", tr.Len(), defaultCap) + } + // Oldest points evicted: index 0 no longer available. + if _, err := tr.RewindTo(0); err == nil { + t.Fatal("expected ErrNoRewindPoint for evicted index") + } else if err != ErrNoRewindPoint { + t.Fatalf("err = %v", err) + } + // Newest point still available. + if _, err := tr.RewindTo(defaultCap + 9); err != nil { + t.Fatalf("newest point missing: %v", err) + } +} + +func TestNoTouchNoPoint(t *testing.T) { + tr, _ := NewTracker("empty", false) + tr.BeginPrompt(0) + if err := tr.EndPrompt(); err != nil { + t.Fatal(err) + } + if tr.Len() != 0 { + t.Fatalf("empty window stored a point: %d", tr.Len()) + } +} + +func TestRewindUnknownIndex(t *testing.T) { + tr, _ := NewTracker("none", false) + if _, err := tr.RewindTo(42); err != ErrNoRewindPoint { + t.Fatalf("err = %v", err) + } +} diff --git a/internal/hunktracker/hunktracker.go b/internal/hunktracker/hunktracker.go new file mode 100644 index 00000000..7062e325 --- /dev/null +++ b/internal/hunktracker/hunktracker.go @@ -0,0 +1,235 @@ +// Package hunktracker tracks line-level edits ("hunks") per file with stable +// hunk identity across re-computation and author attribution that survives +// external edits, adopted from grok-build's xai-hunk-tracker. +// +// The core problem: naive re-diffing after every edit assigns fresh positions +// and IDs, so "the change the agent made" cannot be followed across subsequent +// modifications. Here each hunk carries an ID derived from its content, and +// when file content changes later, previously tracked hunks are matched by +// content-and-overlap so their identity — and their author attribution — is +// preserved even when the newest edit came from someone else. +package hunktracker + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" +) + +// Author classifies who produced a hunk. +type Author string + +const ( + // AuthorAgent marks hunks written by the coding agent. + AuthorAgent Author = "agent" + // AuthorExternal marks hunks written by anything else (user, other tools). + AuthorExternal Author = "external" +) + +// Hunk is one contiguous changed region. +type Hunk struct { + // ID is stable across re-computation while content overlaps. + ID string `json:"id"` + // StartLine is 1-based in the *current* content. + StartLine int `json:"start_line"` + Lines int `json:"lines"` + Text string `json:"text"` + // Author records who last wrote this region. + Author Author `json:"author"` +} + +// FileState is the tracked state of one file. +type FileState struct { + Path string `json:"path"` + Baseline string `json:"baseline"` + Hunks []Hunk `json:"hunks"` +} + +// Tracker holds per-file hunk states. +type Tracker struct { + files map[string]*FileState +} + +// NewTracker returns an empty tracker. +func NewTracker() *Tracker { return &Tracker{files: map[string]*FileState{}} } + +// Track registers a file whose future changes should be attributed. The +// current content becomes the baseline. +func (t *Tracker) Track(path, currentContent string) { + t.files[path] = &FileState{Path: path, Baseline: currentContent} +} + +// Forget removes a file from tracking. +func (t *Tracker) Forget(path string) { delete(t.files, path) } + +// Tracked reports whether path is being tracked. +func (t *Tracker) Tracked(path string) bool { _, ok := t.files[path]; return ok } + +// Update records an edit of a tracked file and returns its hunks after +// identity reconciliation. Untracked paths are ignored (call Track first). +func (t *Tracker) Update(path, newContent string, author Author) ([]Hunk, bool) { + st, ok := t.files[path] + if !ok { + return nil, false + } + newHunks := computeHunks(st.Baseline, newContent) + st.Hunks = reconcile(st.Hunks, newHunks, author) + st.Baseline = newContent + return st.Hunks, true +} + +// Hunks returns a copy of the tracked hunks for a file. +func (t *Tracker) Hunks(path string) []Hunk { + st, ok := t.files[path] + if !ok { + return nil + } + out := make([]Hunk, len(st.Hunks)) + copy(out, st.Hunks) + return out +} + +// AgentTouchedFiles lists tracked files with at least one agent-authored hunk. +func (t *Tracker) AgentTouchedFiles() []string { + var out []string + for p, st := range t.files { + for _, h := range st.Hunks { + if h.Author == AuthorAgent { + out = append(out, p) + break + } + } + } + sort.Strings(out) + return out +} + +// computeHunks diffs baseline vs current line-wise, returning changed regions +// positioned in current. It uses an LCS over lines. +func computeHunks(baseline, current string) []Hunk { + a := splitLines(baseline) + b := splitLines(current) + ops := lcsOps(a, b) // sequence of ops over b indices: keep/delete(insert) + + var hunks []Hunk + i := 0 + for i < len(ops) { + if ops[i] { + // Kept line (part of the LCS with baseline): not a change. + i++ + continue + } + start := i + for i < len(ops) && !ops[i] { + i++ + } + h := Hunk{ + ID: hunkID(b[start:i]), + StartLine: start + 1, + Lines: i - start, + Text: strings.Join(b[start:i], "\n"), + } + hunks = append(hunks, h) + } + return hunks +} + +// reconcile merges newly computed hunks with previously known ones: +// - a new hunk matching an old hunk's ID keeps the old identity and author; +// - a new hunk overlapping an old one keeps the old author (an external edit +// touching an agent-authored region does not strip attribution); +// - brand-new hunks take the incoming author. +func reconcile(old, next []Hunk, author Author) []Hunk { + oldByID := map[string]Hunk{} + for _, h := range old { + oldByID[h.ID] = h + } + out := make([]Hunk, 0, len(next)) + for _, h := range next { + if prev, ok := oldByID[h.ID]; ok { + h.Author = prev.Author + out = append(out, h) + continue + } + attributed := Author("") + for _, prev := range old { + if hunksOverlap(prev, h) { + attributed = prev.Author + break + } + } + if attributed == "" { + attributed = author + } + // Note: an external edit overlapping a previously agent-authored + // region intentionally keeps the prior attribution. + h.Author = attributed + out = append(out, h) + } + return out +} + +func hunksOverlap(a, b Hunk) bool { + aStart, aEnd := a.StartLine, a.StartLine+a.Lines + bStart, bEnd := b.StartLine, b.StartLine+b.Lines + return aStart < bEnd && bStart < aEnd +} + +func hunkID(lines []string) string { + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return fmt.Sprintf("h%s", hex.EncodeToString(sum[:6])) +} + +func splitLines(s string) []string { + s = strings.TrimSuffix(s, "\n") + if s == "" { + return nil + } + return strings.Split(s, "\n") +} + +// lcsOps returns, for each index of b, whether that line is part of the LCS +// with a (true = kept/present in both, false = inserted relative to a). +func lcsOps(a, b []string) []bool { + n, m := len(a), len(b) + // Guard against pathological sizes; fall back to "all inserted". + if n*m > 4_000_000 { + all := make([]bool, m) + for i := range all { + all[i] = true + } + return all + } + dp := make([][]int, n+1) + for i := range dp { + dp[i] = make([]int, m+1) + } + for i := n - 1; i >= 0; i-- { + for j := m - 1; j >= 0; j-- { + if a[i] == b[j] { + dp[i][j] = dp[i+1][j+1] + 1 + } else if dp[i+1][j] >= dp[i][j+1] { + dp[i][j] = dp[i+1][j] + } else { + dp[i][j] = dp[i][j+1] + } + } + } + ops := make([]bool, m) // true = line kept from baseline (LCS member) + i, j := 0, 0 + for i < n && j < m { + switch { + case a[i] == b[j]: + ops[j] = true + i++ + j++ + case dp[i+1][j] >= dp[i][j+1]: + i++ + default: + j++ + } + } + return ops +} diff --git a/internal/hunktracker/hunktracker_test.go b/internal/hunktracker/hunktracker_test.go new file mode 100644 index 00000000..7113e0b7 --- /dev/null +++ b/internal/hunktracker/hunktracker_test.go @@ -0,0 +1,91 @@ +package hunktracker + +import ( + "testing" +) + +func TestComputeHunksBasic(t *testing.T) { + base := "a\nb\nc\nd\ne\n" + cur := "a\nb\nX\nY\nd\ne\n" + hunks := computeHunks(base, cur) + if len(hunks) != 1 { + t.Fatalf("hunks = %+v", hunks) + } + h := hunks[0] + if h.StartLine != 3 || h.Lines != 2 || h.Text != "X\nY" { + t.Fatalf("hunk = %+v", h) + } +} + +func TestTrackerAttributionAndIdentity(t *testing.T) { + tr := NewTracker() + base := "line1\nline2\nline3\n" + tr.Track("f.go", base) + + hs, ok := tr.Update("f.go", "line1\nAGENT\nline3\n", AuthorAgent) + if !ok || len(hs) != 1 || hs[0].Author != AuthorAgent { + t.Fatalf("agent update: %+v ok=%v", hs, ok) + } + + // External edit adjacent to the agent region: the new change hunk overlaps + // the previously agent-authored position, so attribution stays "agent". + hs, _ = tr.Update("f.go", "line1\nTOP\nAGENT\nline3\n", AuthorExternal) + found := false + for _, h := range hs { + if h.StartLine == 2 && h.Lines >= 1 { + found = true + if h.Author != AuthorAgent { + t.Fatalf("agent attribution lost after external edit: %+v", hs) + } + } + } + if !found { + t.Fatalf("expected an overlapping hunk near line 2: %+v", hs) + } +} + +func TestTrackerUntrackedIgnored(t *testing.T) { + tr := NewTracker() + if hs, ok := tr.Update("nope", "x", AuthorAgent); ok || hs != nil { + t.Fatal("untracked file must be ignored") + } +} + +func TestAgentTouchedFiles(t *testing.T) { + tr := NewTracker() + tr.Track("a.go", "1\n") + tr.Track("b.go", "2\n") + if _, ok := tr.Update("a.go", "1\nx\n", AuthorAgent); !ok { + t.Fatal("update a failed") + } + if _, ok := tr.Update("b.go", "2\ny\n", AuthorExternal); !ok { + t.Fatal("update b failed") + } + got := tr.AgentTouchedFiles() + if len(got) != 1 || got[0] != "a.go" { + t.Fatalf("AgentTouchedFiles = %v", got) + } +} + +func TestHunkIDDeterministic(t *testing.T) { + // Identical (baseline, current) pairs must yield identical IDs. + a := computeHunks("x\ny\n", "x\nNEW\ny\n") + b := computeHunks("x\ny\n", "x\nNEW\ny\n") + if len(a) != 1 || len(b) != 1 || a[0].ID != b[0].ID { + t.Fatalf("IDs differ for identical diffs: %v vs %v", a, b) + } +} + +func TestComputeHunksEmptyBaseline(t *testing.T) { + hunks := computeHunks("", "one\ntwo\n") + if len(hunks) != 1 || hunks[0].StartLine != 1 || hunks[0].Lines != 2 { + t.Fatalf("hunks = %+v", hunks) + } +} + +func TestComputeHunksNoChange(t *testing.T) { + s := "a\nb\n" + if hunks := computeHunks(s, s); len(hunks) != 0 { + t.Fatalf("expected no hunks, got %+v", hunks) + } +} diff --git a/internal/tool/file_edit.go b/internal/tool/file_edit.go index f3fb3f0e..6a378a34 100644 --- a/internal/tool/file_edit.go +++ b/internal/tool/file_edit.go @@ -149,7 +149,15 @@ func fuzzyFind(content, oldStr string) (bool, string, float64) { return true, actual, 1.0 } - // Strategy 3: Levenshtein-based similarity matching on contiguous line blocks + // Strategy 3: Unicode-normalized matching (typographic characters). + // Models frequently emit em-dashes for hyphens, smart quotes for straight + // quotes, and non-breaking spaces; folding them back to ASCII recovers + // exact matches that would otherwise fall through to fuzzy similarity. + if matched, actual := unicodeNormalizedFind(content, oldStr); matched { + return true, actual, 1.0 + } + + // Strategy 4: Levenshtein-based similarity matching on contiguous line blocks if matched, actual, sim := levenshteinBlockFind(content, oldStr, 0.90); matched { return true, actual, sim } @@ -157,6 +165,61 @@ func fuzzyFind(content, oldStr string) (bool, string, float64) { return false, "", 0 } +// foldTypographic maps common typographic characters to their ASCII +// equivalents: dashes, quotes, ellipsis, and non-breaking spaces. +func foldTypographic(r rune) rune { + switch r { + case '—', '–', '‑': + return '-' + case '‘', '’', '‚', 'ʼ': + return '\'' + case '“', '”', '„': + return '"' + case '…': + return '.' + case ' ': // U+00A0 non-breaking space + return ' ' + default: + return r + } +} + +// normalizeTypographic folds typographic characters to ASCII in s. +func normalizeTypographic(s string) string { + return strings.Map(foldTypographic, s) +} + +// unicodeNormalizedFind matches oldStr against content after folding +// typographic characters to their ASCII equivalents on both sides. The match +// must be unique in the folded domain. +func unicodeNormalizedFind(content, oldStr string) (bool, string) { + foldedOld := normalizeTypographic(oldStr) + if foldedOld == oldStr { + // No typographic difference in the needle; earlier exact strategies + // already cover this case. + return false, "" + } + foldedContent := normalizeTypographic(content) + idx := strings.Index(foldedContent, foldedOld) + if idx == -1 { + return false, "" + } + if strings.Count(foldedContent, foldedOld) != 1 { + return false, "" + } + + // The fold is 1:1 per rune (every input rune maps to exactly one output + // rune), so rune offsets in foldedContent map directly to rune offsets in + // content. Convert rune offsets to byte offsets. + contentRunes := []rune(content) + startRune := len([]rune(foldedContent[:idx])) + endRune := startRune + len([]rune(foldedOld)) + if endRune > len(contentRunes) { + return false, "" + } + return true, string(contentRunes[startRune:endRune]) +} + // normalizeWhitespace collapses runs of spaces and tabs into a single space. func normalizeWhitespace(s string) string { var b strings.Builder diff --git a/internal/tool/fuzzy_edit_test.go b/internal/tool/fuzzy_edit_test.go index e652f959..078a0b8a 100644 --- a/internal/tool/fuzzy_edit_test.go +++ b/internal/tool/fuzzy_edit_test.go @@ -145,3 +145,37 @@ func TestNormalizeWhitespace(t *testing.T) { } } } + +func TestFuzzyFindUnicodeNormalized(t *testing.T) { + // Model emitted em-dash and smart quotes; file has ASCII. + content := "option = \"fast\" # use the built-in mode\n" + old := "option = \u201cfast\u201d # use the built\u2013in mode\n" + matched, actual, sim := fuzzyFind(content, old) + if !matched { + t.Fatal("unicode-normalized match failed") + } + if actual != content { + t.Fatalf("actual = %q", actual) + } + if sim != 1.0 { + t.Fatalf("sim = %v", sim) + } +} + +func TestUnicodeNormalizedFindRequiresTypographicNeedle(t *testing.T) { + matched, _ := unicodeNormalizedFind("abc", "abc") + if matched { + t.Fatal("plain needle should be skipped (covered by earlier strategies)") + } +} + +func TestFoldTypographic(t *testing.T) { + in := "\u201ca\u201d \u2014 b\u2026 c\u00a0d" + want := "\"a\" - b. cd" // nbsp folds to regular space; trailing join is rune-level + got := normalizeTypographic(in) + // nbsp becomes ' ', so expected includes the space + want = "\"a\" - b. c d" + if got != want { + t.Fatalf("normalizeTypographic = %q, want %q", got, want) + } +} From 0bf46e57109d39823c09825b2ad188c3fc0109f9 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 23 Aug 2026 00:26:57 +0530 Subject: [PATCH 2/2] fix(fastcopy): split clonefile into platform files unix.Clonefile only exists in darwin build tags of golang.org/x/sys, so referencing it from a shared file broke linux/windows builds (caught by CI: module hygiene, vet, race, deadcode, public module graph). Move the APFS clone into clone_darwin.go with a !darwin stub in clone_other.go; verified linux/amd64 and windows/amd64 cross-compiles. --- internal/fastcopy/clone_darwin.go | 21 +++++++++++++++++++++ internal/fastcopy/clone_other.go | 8 ++++++++ internal/fastcopy/fastcopy.go | 19 ++++--------------- 3 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 internal/fastcopy/clone_darwin.go create mode 100644 internal/fastcopy/clone_other.go diff --git a/internal/fastcopy/clone_darwin.go b/internal/fastcopy/clone_darwin.go new file mode 100644 index 00000000..1a3a93c3 --- /dev/null +++ b/internal/fastcopy/clone_darwin.go @@ -0,0 +1,21 @@ +//go:build darwin + +package fastcopy + +import ( + "os" + + "golang.org/x/sys/unix" +) + +// tryCloneFilePlatform clones src into dst via APFS clonefile. +func tryCloneFilePlatform(src, dst string) (int64, bool) { + if err := unix.Clonefile(src, dst, 0); err != nil { + return 0, false + } + info, err := os.Stat(src) + if err != nil { + return 0, false + } + return info.Size(), true +} diff --git a/internal/fastcopy/clone_other.go b/internal/fastcopy/clone_other.go new file mode 100644 index 00000000..efcf1335 --- /dev/null +++ b/internal/fastcopy/clone_other.go @@ -0,0 +1,8 @@ +//go:build !darwin + +package fastcopy + +// tryCloneFilePlatform reports unsupported; callers fall back to byte copies. +func tryCloneFilePlatform(src, dst string) (int64, bool) { + return 0, false +} diff --git a/internal/fastcopy/fastcopy.go b/internal/fastcopy/fastcopy.go index e9214778..6ba17329 100644 --- a/internal/fastcopy/fastcopy.go +++ b/internal/fastcopy/fastcopy.go @@ -19,8 +19,6 @@ import ( "strings" "sync" "sync/atomic" - - "golang.org/x/sys/unix" ) // maxWorkers bounds the copy worker pool. macOS defaults to a low soft @@ -156,20 +154,11 @@ func copyFileCoW(src, dst string) (int64, error) { } // tryCloneFile attempts a filesystem-level clone; returns cloned=false when -// unsupported (caller falls back). APFS clonefile is supported on darwin; -// other platforms take the byte-copy fallback, which stays correct everywhere. +// unsupported (caller falls back to a byte copy, which stays correct +// everywhere). The per-platform implementations live in clone_darwin.go and +// clone_other.go. func tryCloneFile(src, dst string) (int64, bool) { - if runtime.GOOS != "darwin" { - return 0, false - } - if err := unix.Clonefile(src, dst, 0); err != nil { - return 0, false - } - info, err := os.Stat(src) - if err != nil { - return 0, false - } - return info.Size(), true + return tryCloneFilePlatform(src, dst) } // SupportsCloneFile reports whether a quick probe clone succeeds on the