diff --git a/cmd/eval.go b/cmd/eval.go index d354b6bf..2506d8af 100644 --- a/cmd/eval.go +++ b/cmd/eval.go @@ -27,6 +27,7 @@ var ( evalResultsJSON bool evalLoopPrompt string evalLoopModel string + evalLoopReport bool ) var evalCmd = &cobra.Command{ @@ -82,6 +83,7 @@ func init() { evalResultsCmd.Flags().BoolVar(&evalResultsJSON, "json", false, "output results as JSON") evalLoopCmd.Flags().StringVar(&evalLoopPrompt, "prompt", "", "Task prompt to run through the agent loop") evalLoopCmd.Flags().StringVar(&evalLoopModel, "model", "", "Model to use (defaults to active model)") + evalLoopCmd.Flags().BoolVar(&evalLoopReport, "report", false, "print the comparative/reproducibility report") evalCmd.AddCommand(evalRunCmd) evalCmd.AddCommand(evalListCmd) @@ -138,6 +140,13 @@ func runEvalLoop(cmd *cobra.Command, _ []string) error { "cost_usd": result.CostUSD, "duration": result.Duration.String(), "transcript_path": transcriptPath, + "repro_hash": result.ReproHash, + } + if evalLoopReport { + cmp := evalloop.Compare([]evalloop.Result{result}) + if _, err := fmt.Fprintln(cmd.OutOrStdout(), evalloop.FormatComparison(cmp)); err != nil { + return err + } } data, err := json.MarshalIndent(report, "", " ") if err != nil { diff --git a/docs/plans/pi-renderer-and-eval-reporting.md b/docs/plans/pi-renderer-and-eval-reporting.md new file mode 100644 index 00000000..ab392916 --- /dev/null +++ b/docs/plans/pi-renderer-and-eval-reporting.md @@ -0,0 +1,69 @@ +# Pi Adoption Follow-up Plan + +Status: Proposed + +Source: `docs/plans/pi-adoption-plan.md` — the remaining open sub-items after +the three merged Pi PRs (#226, #227, #228). + +## Executive Decision + +Two of the three remaining sub-items are **blocked by the rendering stack** and +cannot be adopted cleanly in hawk's current UI framework: + +- Bubble Tea v2 exposes **no public `Renderer` interface** (its renderer is an + unexported type with internal methods; the only option is `WithoutRenderer`). + A custom differential-renderer swap is therefore not a clean interface + implementation — it would require forking the renderer internals, a major and + non-runtime-verifiable change. +- Kitty render-loop integration depends on that renderer. + +The third sub-item — **agent-runtime eval comparative + reproducibility +reporting** — is fully feasible and safe (non-TUI) and is adopted here. + +## Adopted + +### Agent-runtime eval: comparative + reproducibility reporting + +- Compute a reproducibility hash over each run (prompt, model, provider, + config) so identical runs can be cached and compared. +- Add a comparative report across runs/models: pass rate, token/latency/cost + deltas, and per-run reproducibility hash. +- Wire it into the existing `evalloop` package and `hawk eval loop` output. + +### Scope and ownership + +- Primary: `internal/feature/evalloop`. +- CLI: `cmd/eval.go` loop mode. +- No changes to `internal/sandbox`, `internal/session`, or `internal/daemon`. + +### Required behavior + +1. `Run` records the model, provider, prompt, and a config-version seed. +2. A reproducibility hash (SHA-256) is derived from those inputs plus the + result transcript. +3. A `Compare` helper aggregates multiple results and reports pass-rate and + per-metric deltas (tokens, cost, duration). +4. `hawk eval loop --report` prints the comparative summary. + +### Acceptance criteria + +- Identical inputs produce identical reproducibility hashes. +- The comparative report surfaces token/cost/duration deltas across runs. +- Existing single-run behavior is unchanged. +- Unit tests cover hashing determinism and the comparative report. + +## Deliberately Not Adopted + +- **Bubble Tea renderer integration** — Bubble Tea v2's renderer is unexported; + no clean public seam to swap in a differential renderer. The reusable + line-diff core (`internal/tui/diff`) remains available for a future render + engine or an upstream Bubble Tea change. +- **Kitty render-loop integration** — depends on the above; the encoding library + (`internal/tui/kitty`) remains usable by any renderer that can emit frames. + +## Verification + +- `go test ./...` full suite. +- `make vet`, `make lint`, `hawk verify`. +- Focused `internal/feature/evalloop` tests. +- markdownlint on this document. diff --git a/internal/feature/evalloop/compare.go b/internal/feature/evalloop/compare.go new file mode 100644 index 00000000..72bd1c83 --- /dev/null +++ b/internal/feature/evalloop/compare.go @@ -0,0 +1,124 @@ +package evalloop + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// Inputs identifies the fixed inputs of a run for reproducibility hashing. +type Inputs struct { + Model string `json:"model"` + Provider string `json:"provider"` + Prompt string `json:"prompt"` + // ConfigVersion is bumped whenever loop limits/system-prompt semantics + // change, so a change in configuration invalidates cached hashes. + ConfigVersion int `json:"config_version"` +} + +// ReproHashOf returns a deterministic SHA-256 over the run inputs and the +// transcript. Identical inputs and outputs produce identical hashes. +func ReproHashOf(in Inputs, transcript []byte) string { + payload := struct { + Inputs Inputs `json:"inputs"` + Transcript []byte `json:"transcript"` + }{Inputs: in, Transcript: transcript} + data, err := json.Marshal(payload) + if err != nil { + // JSON of these types cannot fail; fall back to a hash of the inputs. + raw, _ := json.Marshal(in) + data = raw + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// RunRecord is one result annotated with its reproducibility hash. +type RunRecord struct { + Model string `json:"model"` + Provider string `json:"provider"` + Output string `json:"output"` + Tokens int `json:"tokens_used"` + CostUSD float64 `json:"cost_usd"` + Duration string `json:"duration"` + Repro string `json:"repro_hash"` +} + +// Comparison summarizes a set of runs across models/providers. +type Comparison struct { + Runs []RunRecord `json:"runs"` + TotalRuns int `json:"total_runs"` + UniqueHashes int `json:"unique_repro_hashes"` + MinTokens int `json:"min_tokens"` + MaxTokens int `json:"max_tokens"` + MinCostUSD float64 `json:"min_cost_usd"` + MaxCostUSD float64 `json:"max_cost_usd"` + MinDuration time.Duration `json:"min_duration"` + MaxDuration time.Duration `json:"max_duration"` +} + +// Compare aggregates multiple results into a comparative report. It returns a +// stable ordering (by reproducibility hash) so identical runs are adjacent. +func Compare(results []Result) Comparison { + cmp := Comparison{Runs: make([]RunRecord, 0, len(results))} + for _, r := range results { + rec := RunRecord{ + Model: r.Model, Provider: r.Provider, Output: truncate(r.Output, 200), + Tokens: r.TokensUsed, CostUSD: r.CostUSD, + Duration: r.Duration.String(), Repro: r.ReproHash, + } + cmp.Runs = append(cmp.Runs, rec) + } + sort.SliceStable(cmp.Runs, func(i, j int) bool { return cmp.Runs[i].Repro < cmp.Runs[j].Repro }) + + unique := map[string]bool{} + for _, rec := range cmp.Runs { + if rec.Repro != "" { + unique[rec.Repro] = true + } + if cmp.MinTokens == 0 || rec.Tokens < cmp.MinTokens { + cmp.MinTokens = rec.Tokens + } + if rec.Tokens > cmp.MaxTokens { + cmp.MaxTokens = rec.Tokens + } + if cmp.MinCostUSD == 0 || rec.CostUSD < cmp.MinCostUSD { + cmp.MinCostUSD = rec.CostUSD + } + if rec.CostUSD > cmp.MaxCostUSD { + cmp.MaxCostUSD = rec.CostUSD + } + } + cmp.TotalRuns = len(cmp.Runs) + cmp.UniqueHashes = len(unique) + return cmp +} + +// FormatComparison renders a human-readable comparative report. +func FormatComparison(cmp Comparison) string { + var b strings.Builder + fmt.Fprintf(&b, "Runs: %d (unique repro hashes: %d)\n", cmp.TotalRuns, cmp.UniqueHashes) + fmt.Fprintf(&b, "Tokens: %d..%d | Cost USD: %.6f..%.6f\n", cmp.MinTokens, cmp.MaxTokens, cmp.MinCostUSD, cmp.MaxCostUSD) + for _, rec := range cmp.Runs { + fmt.Fprintf(&b, "- %s/%s tokens=%d cost=%.6f dur=%s repro=%s\n", rec.Provider, rec.Model, rec.Tokens, rec.CostUSD, rec.Duration, shortHash(rec.Repro)) + } + return strings.TrimRight(b.String(), "\n") +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + +func shortHash(h string) string { + if len(h) > 12 { + return h[:12] + } + return h +} diff --git a/internal/feature/evalloop/compare_test.go b/internal/feature/evalloop/compare_test.go new file mode 100644 index 00000000..3a11cb5f --- /dev/null +++ b/internal/feature/evalloop/compare_test.go @@ -0,0 +1,68 @@ +package evalloop + +import ( + "strings" + "testing" +) + +func TestReproHashDeterministic(t *testing.T) { + in := Inputs{Model: "m", Provider: "p", Prompt: "task", ConfigVersion: 1} + tx := []byte("transcript") + a := ReproHashOf(in, tx) + b := ReproHashOf(in, tx) + if a != b { + t.Fatalf("hash must be deterministic, got %q vs %q", a, b) + } + if a == "" { + t.Fatal("hash must be non-empty") + } +} + +func TestReproHashChangesOnInput(t *testing.T) { + in := Inputs{Model: "m", Provider: "p", Prompt: "task", ConfigVersion: 1} + base := ReproHashOf(in, []byte("tx")) + if ReproHashOf(in, []byte("other")) == base { + t.Fatal("hash must change when the transcript changes") + } + changed := in + changed.Prompt = "different" + if ReproHashOf(changed, []byte("tx")) == base { + t.Fatal("hash must change when inputs change") + } +} + +func TestCompareAggregatesAndSorts(t *testing.T) { + results := []Result{ + {Model: "m1", Provider: "p", TokensUsed: 100, CostUSD: 0.1, Duration: 1, ReproHash: "b"}, + {Model: "m2", Provider: "p", TokensUsed: 200, CostUSD: 0.3, Duration: 2, ReproHash: "a"}, + } + cmp := Compare(results) + if cmp.TotalRuns != 2 { + t.Fatalf("total runs = %d, want 2", cmp.TotalRuns) + } + if cmp.UniqueHashes != 2 { + t.Fatalf("unique hashes = %d, want 2", cmp.UniqueHashes) + } + if cmp.MinTokens != 100 || cmp.MaxTokens != 200 { + t.Fatalf("token range = %d..%d, want 100..200", cmp.MinTokens, cmp.MaxTokens) + } + if cmp.MinCostUSD != 0.1 || cmp.MaxCostUSD != 0.3 { + t.Fatalf("cost range = %f..%f, want 0.1..0.3", cmp.MinCostUSD, cmp.MaxCostUSD) + } + // Sorted by repro hash: "a" first. + if cmp.Runs[0].Model != "m2" { + t.Fatalf("first run = %s, want m2 (sorted by repro)", cmp.Runs[0].Model) + } +} + +func TestFormatComparison(t *testing.T) { + cmp := Compare([]Result{ + {Model: "m", Provider: "p", TokensUsed: 50, CostUSD: 0.05, Duration: 1, ReproHash: "abc123"}, + }) + out := FormatComparison(cmp) + for _, want := range []string{"Runs: 1", "unique repro hashes: 1", "Tokens: 50", "p/m"} { + if !strings.Contains(out, want) { + t.Errorf("format missing %q: %s", want, out) + } + } +} diff --git a/internal/feature/evalloop/runtime.go b/internal/feature/evalloop/runtime.go index b62f31b9..2df95734 100644 --- a/internal/feature/evalloop/runtime.go +++ b/internal/feature/evalloop/runtime.go @@ -22,6 +22,9 @@ type Event struct { // Result is the outcome of one agent-runtime evaluation. type Result struct { + // Model and Provider identify the backend that produced this run. + Model string `json:"model,omitempty"` + Provider string `json:"provider,omitempty"` // Output is the concatenated assistant output produced by the loop. Output string `json:"output"` // Events are the normalized loop events in order. @@ -34,6 +37,10 @@ type Result struct { Transcript []byte `json:"-"` // Duration is the wall-clock time of the run. Duration time.Duration `json:"duration"` + // ReproHash is a deterministic SHA-256 over the run inputs (model, + // provider, prompt, config version) and the transcript, enabling identical + // runs to be compared or cached. + ReproHash string `json:"repro_hash,omitempty"` } // Runtime executes one agent-runtime evaluation. diff --git a/internal/feature/evalloop/session.go b/internal/feature/evalloop/session.go index c3ba2f8b..104667c0 100644 --- a/internal/feature/evalloop/session.go +++ b/internal/feature/evalloop/session.go @@ -70,6 +70,8 @@ func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Resul } } result.Duration = time.Since(start) + result.Model = r.Model + result.Provider = r.Provider // Snapshot the transcript for offline replay of failing runs. if msgs := sess.Persistence().RawMessages(); msgs != nil { @@ -78,6 +80,12 @@ func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Resul } } + // Reproducibility hash over the fixed inputs and the transcript. + result.ReproHash = ReproHashOf(Inputs{ + Model: r.Model, Provider: r.Provider, Prompt: prompt, + ConfigVersion: evalConfigVersion, + }, result.Transcript) + // Report usage/cost when the backend exposes it via the session cost model. cost := sess.CostValue() if cost != nil { @@ -86,3 +94,7 @@ func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Resul } return result, nil } + +// evalConfigVersion is bumped whenever loop limits or system-prompt semantics +// change, so reproducibility hashes are invalidated across versions. +const evalConfigVersion = 1