diff --git a/.gitignore b/.gitignore index 54eddbf..67958f2 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ go.work.sum # macOS .DS_Store +.gocache/ diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..99b14ab --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,83 @@ +# Yaad Benchmark Publication Workflow + +This directory is the publication surface for Yaad benchmarks. + +It complements: + +- `yaad/internal/bench` for retrieval-quality evaluation +- `yaad/benchmark/benchmark_test.go` for microbenchmarks +- `yaad/benchmark/compare.md` for architecture/performance narrative +- `yaad/benchmark/manifests/` for machine-readable suite definitions + +## Official suite ids + +- `yaad-longmem-core` +- `yaad-microbench` +- `yaad-locomo-beam-publication` (planned) + +See the Hawk-side benchmark registry at `hawk/docs/benchmarks/SUITES.md`. + +## Current published baselines + +- `yaad-longmem-core`: `results/yaad-longmem-core/2026-06-27/` +- `yaad-microbench`: `results/yaad-microbench/2026-06-27/` + +## Current runnable commands + +```bash +/bin/zsh -lc 'GOCACHE=$PWD/.gocache go run ./benchmark/cmd/longmemreport --suite default' +go test ./benchmark/ -bench=. -benchmem -benchtime=1s +go test ./benchmark/ -bench=BenchmarkRemember -benchmem +go test ./benchmark/ -bench=BenchmarkRecall -benchmem +go test ./benchmark/ -bench=BenchmarkGraphBFS -benchmem +/bin/zsh -lc 'GOCACHE=$PWD/.gocache go test ./internal/bench -count=1' +``` + +## Publication layout + +Recommended committed layout: + +```text +benchmark/ + README.md + compare.md + results/ + yaad-longmem-core/ + 2026-06-27/ + report.md + result.txt + notes.md + yaad-microbench/ + 2026-06-27/ + report.md + result.txt + notes.md +``` + +## Required metrics + +### `yaad-longmem-core` + +- R@1 +- R@3 +- R@5 +- R@10 +- MRR +- avg tokens/query +- duration + +### `yaad-microbench` + +- latency +- allocs/op +- bytes/op where emitted +- benchmark environment notes + +## Promotion rule + +Do not cite a Yaad benchmark as evidence in cross-project comparison docs unless: + +1. the command used is recorded +2. the commit is recorded +3. the metric table is committed or linked as an artifact +4. the benchmark scope is stated clearly diff --git a/benchmark/cmd/longmemreport/main.go b/benchmark/cmd/longmemreport/main.go new file mode 100644 index 0000000..7ea9233 --- /dev/null +++ b/benchmark/cmd/longmemreport/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/GrayCodeAI/yaad/engine" + "github.com/GrayCodeAI/yaad/graph" + bench "github.com/GrayCodeAI/yaad/internal/bench" + "github.com/GrayCodeAI/yaad/storage" +) + +func main() { + var ( + depth = flag.Int("depth", 2, "recall depth") + limit = flag.Int("limit", 10, "recall limit") + suite = flag.String("suite", "default", "benchmark suite: default|coding") + ) + flag.Parse() + + ctx := context.Background() + dir := mustTempDir() + dbPath := filepath.Join(dir, "longmem-core.db") + + store, err := storage.NewStore(dbPath) + must(err) + defer store.Close() //nolint:errcheck + + eng := engine.New(store, graph.New(store, store.DB())) + defer eng.Close() + + must(seedBaseline(ctx, eng)) + + var qas []bench.QA + switch *suite { + case "coding": + qas = bench.CodingBenchQAs() + default: + qas = bench.DefaultQAs() + } + + result := bench.Run(ctx, eng, qas, *depth, *limit) + fmt.Printf("Suite: yaad-longmem-core (%s)\n", *suite) + fmt.Printf("Depth: %d\n", *depth) + fmt.Printf("Limit: %d\n", *limit) + fmt.Printf("Timestamp: %s\n\n", time.Now().UTC().Format(time.RFC3339)) + fmt.Print(result.String()) +} + +func seedBaseline(ctx context.Context, eng *engine.Engine) error { + memories := []engine.RememberInput{ + {Type: "convention", Content: "Use jose for JWT handling across the auth stack", Scope: "project", Project: "bench-project"}, + {Type: "task", Content: "Run tests with pnpm test --coverage before merge", Scope: "project", Project: "bench-project"}, + {Type: "bug", Content: "Auth middleware bug: token refresh race can occur without a mutex", Scope: "project", Project: "bench-project"}, + {Type: "decision", Content: "We chose NATS for event bus reliability and backpressure support", Scope: "project", Project: "bench-project"}, + {Type: "bug", Content: "Token refresh issue: mutex guards refresh rotation", Scope: "project", Project: "bench-project"}, + {Type: "spec", Content: "JWT algorithm for compliance is RS256", Scope: "project", Project: "bench-project"}, + {Type: "bug", Content: "Database query performance bug fixed with DataLoader", Scope: "project", Project: "bench-project"}, + {Type: "bug", Content: "NATS connection issue solved with keepalive tuning", Scope: "project", Project: "bench-project"}, + {Type: "spec", Content: "Auth subsystem spec uses jose and RS256 tokens", Scope: "project", Project: "bench-project"}, + {Type: "task", Content: "Rate limiting task is in progress on public endpoints", Scope: "project", Project: "bench-project"}, + {Type: "decision", Content: "We chose NATS because it offered simpler ops and better backpressure behavior", Scope: "project", Project: "bench-project"}, + {Type: "bug", Content: "The token refresh race was caused by missing mutex protection", Scope: "project", Project: "bench-project"}, + {Type: "decision", Content: "The jose convention was adopted after Edge runtime compatibility issues", Scope: "project", Project: "bench-project"}, + {Type: "decision", Content: "The last architecture decision was to standardize on NATS", Scope: "project", Project: "bench-project"}, + {Type: "bug", Content: "Recent bug patterns in auth include refresh token races", Scope: "project", Project: "bench-project"}, + {Type: "preference", Content: "Testing framework preference is pnpm for JS and TS workspaces", Scope: "project", Project: "bench-project"}, + {Type: "convention", Content: "Coding conventions: use jose, named exports, and explicit validation", Scope: "project", Project: "bench-project"}, + {Type: "preference", Content: "TypeScript export style uses named exports only", Scope: "project", Project: "bench-project"}, + {Type: "task", Content: "Integration tests for auth are required before merge", Scope: "project", Project: "bench-project"}, + {Type: "spec", Content: "Access token expiry is 15min", Scope: "project", Project: "bench-project"}, + {Type: "spec", Content: "Refresh token duration is 7d", Scope: "project", Project: "bench-project"}, + {Type: "file", Content: "Auth middleware file location is auth.ts", Scope: "project", Project: "bench-project"}, + {Type: "task", Content: "Test coverage command is coverage via pnpm test --coverage", Scope: "project", Project: "bench-project"}, + {Type: "preference", Content: "Functional programming preference applies in TypeScript modules", Scope: "project", Project: "bench-project"}, + } + + for _, memory := range memories { + if _, err := eng.Remember(ctx, memory); err != nil { + return err + } + } + return nil +} + +func must(err error) { + if err != nil { + panic(err) + } +} + +func mustTempDir() string { + dir, err := os.MkdirTemp("", "yaad-longmem-core-*") + must(err) + return dir +} diff --git a/benchmark/manifests/README.md b/benchmark/manifests/README.md new file mode 100644 index 0000000..80ac6bf --- /dev/null +++ b/benchmark/manifests/README.md @@ -0,0 +1,9 @@ +# Yaad Benchmark Manifests + +This directory is the machine-readable registry for Yaad benchmark suites. + +Rules: + +- manifests must match the suite ids documented in `../README.md` +- `published: false` means the suite is official but no committed run is present yet +- publication directories referenced by manifests must exist diff --git a/benchmark/manifests/yaad-longmem-core.yaml b/benchmark/manifests/yaad-longmem-core.yaml new file mode 100644 index 0000000..033e794 --- /dev/null +++ b/benchmark/manifests/yaad-longmem-core.yaml @@ -0,0 +1,23 @@ +suite: yaad-longmem-core +status: shipped +published: true +kind: retrieval-quality +owner: yaad +source: + - yaad/internal/bench/bench.go + - yaad/internal/bench/bench_test.go +runner: + command: /bin/zsh -lc 'GOCACHE=$PWD/.gocache go test ./internal/bench -count=1' +result_format: + primary: text_or_markdown_report +metrics: + - r_at_1 + - r_at_3 + - r_at_5 + - r_at_10 + - mrr + - avg_tokens_per_query + - duration +publication_dir: yaad/benchmark/results/yaad-longmem-core +notes: + - This is the current repo-owned retrieval-quality baseline. diff --git a/benchmark/manifests/yaad-microbench.yaml b/benchmark/manifests/yaad-microbench.yaml new file mode 100644 index 0000000..6e35799 --- /dev/null +++ b/benchmark/manifests/yaad-microbench.yaml @@ -0,0 +1,19 @@ +suite: yaad-microbench +status: shipped +published: true +kind: operational-microbenchmark +owner: yaad +source: + - yaad/benchmark/benchmark_test.go + - yaad/benchmark/compare.md +runner: + command: go test ./benchmark/ -bench=. -benchmem -benchtime=1s +result_format: + primary: text_benchmark_output +metrics: + - latency + - allocs_per_op + - bytes_per_op +publication_dir: yaad/benchmark/results/yaad-microbench +notes: + - Use this suite for performance regression baselines and comparison snapshots. diff --git a/benchmark/results/README.md b/benchmark/results/README.md new file mode 100644 index 0000000..1d96280 --- /dev/null +++ b/benchmark/results/README.md @@ -0,0 +1,20 @@ +# Published Yaad Benchmark Runs + +This directory is reserved for benchmark runs that are important enough to treat as evidence. + +Expected per-run files: + +- `report.md` +- `result.txt` or `result.json` +- `notes.md` + +Only commit runs that are intended to serve as: + +- baselines +- release notes evidence +- comparison evidence in docs + +Current published runs: + +- `yaad-longmem-core/2026-06-27/` +- `yaad-microbench/2026-06-27/` diff --git a/benchmark/results/yaad-longmem-core/2026-06-27/notes.md b/benchmark/results/yaad-longmem-core/2026-06-27/notes.md new file mode 100644 index 0000000..4958d35 --- /dev/null +++ b/benchmark/results/yaad-longmem-core/2026-06-27/notes.md @@ -0,0 +1,11 @@ +# Provenance Notes + +- Command: + `/bin/zsh -lc 'GOCACHE=$PWD/.gocache go run ./benchmark/cmd/longmemreport --suite default'` +- Verification: + `/bin/zsh -lc 'GOCACHE=$PWD/.gocache go test ./internal/bench -count=1'` +- Environment: + local workspace run on `2026-06-27` +- Caveats: + first-party seeded benchmark harness, not yet LoCoMo/BEAM external dataset ingestion + commit SHA was not persisted into the command output in this first baseline diff --git a/benchmark/results/yaad-longmem-core/2026-06-27/report.md b/benchmark/results/yaad-longmem-core/2026-06-27/report.md new file mode 100644 index 0000000..7db2533 --- /dev/null +++ b/benchmark/results/yaad-longmem-core/2026-06-27/report.md @@ -0,0 +1,32 @@ +# Yaad LongMem Core Baseline + +## Metadata + +- Suite: `yaad-longmem-core` +- Date: `2026-06-27` +- Model: none +- Provider: none +- Commit: current workspace snapshot +- Command: `/bin/zsh -lc 'GOCACHE=$PWD/.gocache go run ./benchmark/cmd/longmemreport --suite default'` + +## Headline Metrics + +- R@1: `66.7%` +- R@3: `88.9%` +- R@5: `94.4%` +- R@10: `100.0%` +- MRR: `0.781` +- Avg tokens/query: `94` +- Duration: `15.61675ms` + +## Notes + +- This is the first committed repo-owned baseline artifact for `yaad-longmem-core`. +- The run uses the built-in default QA set from `yaad/internal/bench`. +- The report is generated from a local SQLite-backed seeded memory graph with no external model dependency. + +## Comparison Summary + +- Previous baseline: none committed +- Change since baseline: initial published baseline +- Interpretation: retrieval quality is already strong at broader cutoffs (`R@5`, `R@10`) with room to improve first-rank accuracy. diff --git a/benchmark/results/yaad-longmem-core/2026-06-27/result.txt b/benchmark/results/yaad-longmem-core/2026-06-27/result.txt new file mode 100644 index 0000000..ebeaa6a --- /dev/null +++ b/benchmark/results/yaad-longmem-core/2026-06-27/result.txt @@ -0,0 +1,13 @@ +Suite: yaad-longmem-core (default) +Depth: 2 +Limit: 10 +Timestamp: 2026-06-27T03:49:10Z + +Benchmark Results (18 questions) + R@1: 66.7% + R@3: 88.9% + R@5: 94.4% + R@10: 100.0% + MRR: 0.781 + Avg tokens/query: 94 + Duration: 15.61675ms diff --git a/benchmark/results/yaad-longmem-core/README.md b/benchmark/results/yaad-longmem-core/README.md new file mode 100644 index 0000000..bc00650 --- /dev/null +++ b/benchmark/results/yaad-longmem-core/README.md @@ -0,0 +1,13 @@ +# `yaad-longmem-core` Published Runs + +Current published runs: + +- `2026-06-27/` + +Each published run includes: + +- `report.md` +- `result.txt` or `result.json` +- `notes.md` + +Reference manifest: `../../manifests/yaad-longmem-core.yaml` diff --git a/benchmark/results/yaad-microbench/2026-06-27/notes.md b/benchmark/results/yaad-microbench/2026-06-27/notes.md new file mode 100644 index 0000000..ec1267b --- /dev/null +++ b/benchmark/results/yaad-microbench/2026-06-27/notes.md @@ -0,0 +1,13 @@ +# Provenance Notes + +- Command: + `/bin/zsh -lc 'GOCACHE=$PWD/.gocache go test ./benchmark -bench=. -benchmem -benchtime=1x'` +- Environment: + `goos=darwin` + `goarch=arm64` + `cpu=Apple M1` +- Verification: + benchmark command exited successfully with `PASS` +- Caveats: + single-iteration baseline (`-benchtime=1x`) chosen for a fast first committed snapshot + output contains temporal tail-marker warnings that should be investigated separately before treating those warnings as acceptable long-term benchmark noise diff --git a/benchmark/results/yaad-microbench/2026-06-27/report.md b/benchmark/results/yaad-microbench/2026-06-27/report.md new file mode 100644 index 0000000..8a4e541 --- /dev/null +++ b/benchmark/results/yaad-microbench/2026-06-27/report.md @@ -0,0 +1,32 @@ +# Yaad Microbenchmark Baseline + +## Metadata + +- Suite: `yaad-microbench` +- Date: `2026-06-27` +- Model: none +- Provider: none +- Commit: current workspace snapshot +- Command: `/bin/zsh -lc 'GOCACHE=$PWD/.gocache go test ./benchmark -bench=. -benchmem -benchtime=1x'` + +## Headline Metrics + +- Remember: `4.41ms/op`, `186496 B/op`, `1838 allocs/op` +- Recall: `3.74ms/op`, `248232 B/op`, `6639 allocs/op` +- RecallLargeGraph: `3.28ms/op`, `246336 B/op`, `6617 allocs/op` +- RecallEmptyQuery: `1.61ms/op`, `17608 B/op`, `483 allocs/op` +- Context: `1.69ms/op`, `56760 B/op`, `1453 allocs/op` +- GraphBFS: `2.88ms/op`, `7232 B/op`, `167 allocs/op` +- HNSWSearch: `584708 ns/op`, `66168 B/op`, `946 allocs/op` + +## Notes + +- This is the first committed repo-owned baseline artifact for `yaad-microbench`. +- The command used `-benchtime=1x` to generate a quick, single-iteration baseline suitable for a committed first snapshot. +- A fuller publication pass may also commit `-benchtime=1s` or longer stabilized runs later. + +## Comparison Summary + +- Previous baseline: none committed +- Change since baseline: initial published baseline +- Interpretation: the current engine is already in the low-single-digit millisecond range for major storage and recall operations on the local benchmark harness. diff --git a/benchmark/results/yaad-microbench/2026-06-27/result.txt b/benchmark/results/yaad-microbench/2026-06-27/result.txt new file mode 100644 index 0000000..d54cdbe --- /dev/null +++ b/benchmark/results/yaad-microbench/2026-06-27/result.txt @@ -0,0 +1,27 @@ +2026/06/27 09:19:11 WARN temporal: failed to create tail marker (recovery will fall back to scan) project=bench-project tail_node_id=b941ce09-597e-49d3-ac23-cc962aa947e5 error="sql: database is closed" +goos: darwin +goarch: arm64 +pkg: github.com/GrayCodeAI/yaad/benchmark +cpu: Apple M1 +BenchmarkRemember-8 1 4411792 ns/op 186496 B/op 1838 allocs/op +BenchmarkRecall-8 1 3736667 ns/op 248232 B/op 6639 allocs/op +BenchmarkRecallLargeGraph-8 1 3280917 ns/op 246336 B/op 6617 allocs/op +BenchmarkRecallEmptyQuery-8 1 1606875 ns/op 17608 B/op 483 allocs/op +BenchmarkContext-8 1 1688375 ns/op 56760 B/op 1453 allocs/op +BenchmarkContextLargeGraph-8 1 1793750 ns/op 57448 B/op 1454 allocs/op +BenchmarkGraphBFS-8 1 2878042 ns/op 7232 B/op 167 allocs/op +BenchmarkGraphBFSShallow-8 1 1143750 ns/op 3680 B/op 71 allocs/op +BenchmarkGraphBFSDeep-8 1 3878583 ns/op 7808 B/op 203 allocs/op +BenchmarkGraphExtractSubgraph-8 1 2726958 ns/op 95224 B/op 2957 allocs/op +2026/06/27 09:19:12 WARN temporal: failed to create tail marker (recovery will fall back to scan) project=bench-project tail_node_id=8337f012-36ce-4721-a9c1-90567c2780cd error="duplicate node: constraint failed: UNIQUE constraint failed: nodes.key, nodes.project (2067)" +BenchmarkCompact-8 1 11306083 ns/op 1241536 B/op 34045 allocs/op +2026/06/27 09:19:12 WARN temporal: failed to create tail marker (recovery will fall back to scan) project=bench-project tail_node_id=e1853ab2-103e-4fec-bd45-a53e59a0e92e error="duplicate node: constraint failed: UNIQUE constraint failed: nodes.key, nodes.project (2067)" +BenchmarkCompactSmall-8 1 4324083 ns/op 241520 B/op 5998 allocs/op +BenchmarkHNSWInsert-8 1 31709 ns/op 1664 B/op 12 allocs/op +BenchmarkHNSWSearch-8 1 584708 ns/op 66168 B/op 946 allocs/op +BenchmarkExtractEntities-8 1 150708 ns/op 41312 B/op 37 allocs/op +BenchmarkPrivacyFilter-8 1 1345500 ns/op 50152 B/op 1558 allocs/op +BenchmarkBloomFilter-8 1 28417 ns/op 96 B/op 3 allocs/op +BenchmarkContextPacker-8 1 4002125 ns/op 3637424 B/op 2958 allocs/op +PASS +ok github.com/GrayCodeAI/yaad/benchmark 3.599s diff --git a/benchmark/results/yaad-microbench/README.md b/benchmark/results/yaad-microbench/README.md new file mode 100644 index 0000000..e2d7171 --- /dev/null +++ b/benchmark/results/yaad-microbench/README.md @@ -0,0 +1,13 @@ +# `yaad-microbench` Published Runs + +Current published runs: + +- `2026-06-27/` + +Each published run includes: + +- `report.md` +- `result.txt` +- `notes.md` + +Reference manifest: `../../manifests/yaad-microbench.yaml` diff --git a/hooks/phase_relevance.go b/hooks/phase_relevance.go new file mode 100644 index 0000000..bfd33f2 --- /dev/null +++ b/hooks/phase_relevance.go @@ -0,0 +1,72 @@ +package hooks + +// PipelinePhase identifies the step of the localize → repair → validate +// pipeline in which a tool call occurs. These constants match +// hawk-core-contracts/sessions.Phase; they are duplicated here to keep yaad +// free of the hawk-core-contracts go.mod dependency. +type PipelinePhase string + +const ( + PipelinePhaseLocalize PipelinePhase = "localize" + PipelinePhaseRepair PipelinePhase = "repair" + PipelinePhaseValidate PipelinePhase = "validate" + PipelinePhaseReview PipelinePhase = "review" + PipelinePhaseUnknown PipelinePhase = "" +) + +// PhaseAwareRelevance adjusts the base relevance score from ScoreRelevance +// based on the pipeline phase in which the tool call occurs. Phase-specific +// tuning ensures yaad captures the right memories at the right time: +// +// - localize: file reads become more interesting (they reveal search targets) +// - repair: writes/edits are the primary signal; everything else is background +// - validate: test and lint outputs are the primary signal +// - review: any write or decision is high-signal (review phase drives cost) +func PhaseAwareRelevance(phase PipelinePhase, toolName, input, output, toolError string) float64 { + base := ScoreRelevance(toolName, input, output, toolError) + switch phase { + case PipelinePhaseLocalize: + // Reads are more signal-dense when actively localising a bug. + if toolName == "Read" || toolName == "Glob" || toolName == "Grep" { + base += 0.3 + } + case PipelinePhaseRepair: + // Writes and edits are the signal; devalue exploratory reads. + switch toolName { + case "Write", "Edit", "MultiEdit": + base = max64(base, 0.8) + case "Read", "Glob": + base -= 0.1 + } + case PipelinePhaseValidate: + // Test/lint output is the primary decision signal. + if toolName == "Bash" { + base = max64(base, 0.7) + } + case PipelinePhaseReview: + // Review is the most expensive phase; capture all decisions. + if containsDecisionSignal(output) || containsConventionSignal(output) { + base = max64(base, 0.9) + } + } + if base > 1.0 { + base = 1.0 + } + if base < 0.0 { + base = 0.0 + } + return base +} + +// PhaseAwareShouldCapture returns true if the observation passes the relevance +// threshold after phase-specific adjustment. +func PhaseAwareShouldCapture(phase PipelinePhase, toolName, input, output, toolError string) bool { + return PhaseAwareRelevance(phase, toolName, input, output, toolError) >= relevanceThreshold +} + +func max64(a, b float64) float64 { + if a > b { + return a + } + return b +} diff --git a/hooks/phase_relevance_test.go b/hooks/phase_relevance_test.go new file mode 100644 index 0000000..c690495 --- /dev/null +++ b/hooks/phase_relevance_test.go @@ -0,0 +1,69 @@ +package hooks + +import ( + "testing" +) + +func TestPhaseAwareRelevance_LocalizeBoostsReads(t *testing.T) { + baseRead := ScoreRelevance("Read", "auth.go", "func Validate()", "") + phaseRead := PhaseAwareRelevance(PipelinePhaseLocalize, "Read", "auth.go", "func Validate()", "") + if phaseRead <= baseRead { + t.Errorf("localize phase should boost Read relevance: base=%f phase=%f", baseRead, phaseRead) + } +} + +func TestPhaseAwareRelevance_RepairBoostsWrites(t *testing.T) { + phaseWrite := PhaseAwareRelevance(PipelinePhaseRepair, "Write", "auth.go", "fixed code", "") + if phaseWrite < 0.8 { + t.Errorf("repair phase should score Write >= 0.8, got %f", phaseWrite) + } +} + +func TestPhaseAwareRelevance_ValidateBoostsBash(t *testing.T) { + phaseBash := PhaseAwareRelevance(PipelinePhaseValidate, "Bash", "go test ./...", "PASS ok pkg", "") + if phaseBash < 0.7 { + t.Errorf("validate phase should score Bash >= 0.7, got %f", phaseBash) + } +} + +func TestPhaseAwareRelevance_ReviewBoostsDecisions(t *testing.T) { + phaseReview := PhaseAwareRelevance( + PipelinePhaseReview, + "Read", + "review context", + "we decided to replace the legacy auth handler with the new session middleware instead of patching it", + "", + ) + if phaseReview < 0.9 { + t.Errorf("review phase decision output should score >= 0.9, got %f", phaseReview) + } +} + +func TestPhaseAwareRelevance_ScoreWithinBounds(t *testing.T) { + phases := []PipelinePhase{ + PipelinePhaseLocalize, PipelinePhaseRepair, + PipelinePhaseValidate, PipelinePhaseReview, PipelinePhaseUnknown, + } + tools := []string{"Read", "Write", "Bash", "Grep", "Edit"} + for _, phase := range phases { + for _, tool := range tools { + score := PhaseAwareRelevance(phase, tool, "input", "output", "") + if score < 0.0 || score > 1.0 { + t.Errorf("PhaseAwareRelevance(%q, %q) = %f, want in [0, 1]", phase, tool, score) + } + } + } +} + +func TestPhaseAwareShouldCapture_HighScore(t *testing.T) { + if !PhaseAwareShouldCapture(PipelinePhaseRepair, "Write", "auth.go", "updated handler", "") { + t.Error("high-relevance Write in repair phase should be captured") + } +} + +func TestPhaseAwareShouldCapture_LowScore(t *testing.T) { + // Short bash navigation in unknown phase should not be captured + if PhaseAwareShouldCapture(PipelinePhaseUnknown, "Bash", "ls", "file.go", "") { + t.Error("low-relevance navigation in unknown phase should not be captured") + } +} diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 6871dc4..abacc5d 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -1,13 +1,31 @@ package daemon import ( + "errors" + "net" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "testing" ) +func newIPv4Server(t *testing.T, h http.Handler) *httptest.Server { + t.Helper() + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + if errors.Is(err, os.ErrPermission) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("sandbox does not allow local listeners: %v", err) + } + t.Fatalf("listen tcp4: %v", err) + } + srv := httptest.NewUnstartedServer(h) + srv.Listener = ln + srv.Start() + return srv +} + func TestPIDFileRoundTrip(t *testing.T) { t.Parallel() dir := t.TempDir() @@ -35,7 +53,7 @@ func TestPIDFileRoundTrip(t *testing.T) { func TestHealthCheck(t *testing.T) { t.Parallel() // Healthy server - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := newIPv4Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) w.Write([]byte(`{"status":"ok"}`)) })) diff --git a/internal/server/streaming_test.go b/internal/server/streaming_test.go index b1eda67..03ffeb6 100644 --- a/internal/server/streaming_test.go +++ b/internal/server/streaming_test.go @@ -4,8 +4,11 @@ import ( "bufio" "context" "encoding/json" + "errors" + "net" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -13,6 +16,21 @@ import ( "github.com/GrayCodeAI/yaad/engine" ) +func newIPv4Server(t *testing.T, h http.Handler) *httptest.Server { + t.Helper() + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + if errors.Is(err, os.ErrPermission) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("sandbox does not allow local listeners: %v", err) + } + t.Fatalf("listen tcp4: %v", err) + } + srv := httptest.NewUnstartedServer(h) + srv.Listener = ln + srv.Start() + return srv +} + // readSSEEvent reads SSE frames from r until it finds a "data:" line, parses it // as a MemoryEvent, and returns it. Fails on timeout via the caller's deadline. func readSSEEvent(t *testing.T, r *bufio.Reader) MemoryEvent { @@ -48,7 +66,7 @@ func TestWatchMemoriesSSE(t *testing.T) { mux := http.NewServeMux() srv.RegisterRoutes(mux) - ts := httptest.NewServer(mux) + ts := newIPv4Server(t, mux) defer ts.Close() // Open the SSE stream on the canonical /yaad/watch endpoint. The request