feat(pi): eval reproducibility and comparative reporting

cmd/eval.go

@@ -27,6 +27,7 @@ var (
 	evalResultsJSON bool
 	evalLoopPrompt  string
 	evalLoopModel   string
+	evalLoopReport  bool
 )
 
 var evalCmd = &cobra.Command{
@@ -82,6 +83,7 @@ func init() {
 	evalResultsCmd.Flags().BoolVar(&evalResultsJSON, "json", false, "output results as JSON")
 	evalLoopCmd.Flags().StringVar(&evalLoopPrompt, "prompt", "", "Task prompt to run through the agent loop")
 	evalLoopCmd.Flags().StringVar(&evalLoopModel, "model", "", "Model to use (defaults to active model)")
+	evalLoopCmd.Flags().BoolVar(&evalLoopReport, "report", false, "print the comparative/reproducibility report")
 
 	evalCmd.AddCommand(evalRunCmd)
 	evalCmd.AddCommand(evalListCmd)
@@ -138,6 +140,13 @@ func runEvalLoop(cmd *cobra.Command, _ []string) error {
 		"cost_usd":        result.CostUSD,
 		"duration":        result.Duration.String(),
 		"transcript_path": transcriptPath,
+		"repro_hash":      result.ReproHash,
+	}
+	if evalLoopReport {
+		cmp := evalloop.Compare([]evalloop.Result{result})
+		if _, err := fmt.Fprintln(cmd.OutOrStdout(), evalloop.FormatComparison(cmp)); err != nil {
+			return err
+		}
 	}
 	data, err := json.MarshalIndent(report, "", "  ")
 	if err != nil {

docs/plans/pi-renderer-and-eval-reporting.md

@@ -0,0 +1,69 @@
+# Pi Adoption Follow-up Plan
+
+Status: Proposed
+
+Source: `docs/plans/pi-adoption-plan.md` — the remaining open sub-items after
+the three merged Pi PRs (#226, #227, #228).
+
+## Executive Decision
+
+Two of the three remaining sub-items are **blocked by the rendering stack** and
+cannot be adopted cleanly in hawk's current UI framework:
+
+- Bubble Tea v2 exposes **no public `Renderer` interface** (its renderer is an
+  unexported type with internal methods; the only option is `WithoutRenderer`).
+  A custom differential-renderer swap is therefore not a clean interface
+  implementation — it would require forking the renderer internals, a major and
+  non-runtime-verifiable change.
+- Kitty render-loop integration depends on that renderer.
+
+The third sub-item — **agent-runtime eval comparative + reproducibility
+reporting** — is fully feasible and safe (non-TUI) and is adopted here.
+
+## Adopted
+
+### Agent-runtime eval: comparative + reproducibility reporting
+
+- Compute a reproducibility hash over each run (prompt, model, provider,
+  config) so identical runs can be cached and compared.
+- Add a comparative report across runs/models: pass rate, token/latency/cost
+  deltas, and per-run reproducibility hash.
+- Wire it into the existing `evalloop` package and `hawk eval loop` output.
+
+### Scope and ownership
+
+- Primary: `internal/feature/evalloop`.
+- CLI: `cmd/eval.go` loop mode.
+- No changes to `internal/sandbox`, `internal/session`, or `internal/daemon`.
+
+### Required behavior
+
+1. `Run` records the model, provider, prompt, and a config-version seed.
+2. A reproducibility hash (SHA-256) is derived from those inputs plus the
+   result transcript.
+3. A `Compare` helper aggregates multiple results and reports pass-rate and
+   per-metric deltas (tokens, cost, duration).
+4. `hawk eval loop --report` prints the comparative summary.
+
+### Acceptance criteria
+
+- Identical inputs produce identical reproducibility hashes.
+- The comparative report surfaces token/cost/duration deltas across runs.
+- Existing single-run behavior is unchanged.
+- Unit tests cover hashing determinism and the comparative report.
+
+## Deliberately Not Adopted
+
+- **Bubble Tea renderer integration** — Bubble Tea v2's renderer is unexported;
+  no clean public seam to swap in a differential renderer. The reusable
+  line-diff core (`internal/tui/diff`) remains available for a future render
+  engine or an upstream Bubble Tea change.
+- **Kitty render-loop integration** — depends on the above; the encoding library
+  (`internal/tui/kitty`) remains usable by any renderer that can emit frames.
+
+## Verification
+
+- `go test ./...` full suite.
+- `make vet`, `make lint`, `hawk verify`.
+- Focused `internal/feature/evalloop` tests.
+- markdownlint on this document.

internal/feature/evalloop/compare.go

@@ -0,0 +1,124 @@
+package evalloop
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Inputs identifies the fixed inputs of a run for reproducibility hashing.
+type Inputs struct {
+	Model    string `json:"model"`
+	Provider string `json:"provider"`
+	Prompt   string `json:"prompt"`
+	// ConfigVersion is bumped whenever loop limits/system-prompt semantics
+	// change, so a change in configuration invalidates cached hashes.
+	ConfigVersion int `json:"config_version"`
+}
+
+// ReproHashOf returns a deterministic SHA-256 over the run inputs and the
+// transcript. Identical inputs and outputs produce identical hashes.
+func ReproHashOf(in Inputs, transcript []byte) string {
+	payload := struct {
+		Inputs     Inputs `json:"inputs"`
+		Transcript []byte `json:"transcript"`
+	}{Inputs: in, Transcript: transcript}
+	data, err := json.Marshal(payload)
+	if err != nil {
+		// JSON of these types cannot fail; fall back to a hash of the inputs.
+		raw, _ := json.Marshal(in)
+		data = raw
+	}
+	sum := sha256.Sum256(data)
+	return hex.EncodeToString(sum[:])
+}
+
+// RunRecord is one result annotated with its reproducibility hash.
+type RunRecord struct {
+	Model    string  `json:"model"`
+	Provider string  `json:"provider"`
+	Output   string  `json:"output"`
+	Tokens   int     `json:"tokens_used"`
+	CostUSD  float64 `json:"cost_usd"`
+	Duration string  `json:"duration"`
+	Repro    string  `json:"repro_hash"`
+}
+
+// Comparison summarizes a set of runs across models/providers.
+type Comparison struct {
+	Runs         []RunRecord   `json:"runs"`
+	TotalRuns    int           `json:"total_runs"`
+	UniqueHashes int           `json:"unique_repro_hashes"`
+	MinTokens    int           `json:"min_tokens"`
+	MaxTokens    int           `json:"max_tokens"`
+	MinCostUSD   float64       `json:"min_cost_usd"`
+	MaxCostUSD   float64       `json:"max_cost_usd"`
+	MinDuration  time.Duration `json:"min_duration"`
+	MaxDuration  time.Duration `json:"max_duration"`
+}
+
+// Compare aggregates multiple results into a comparative report. It returns a
+// stable ordering (by reproducibility hash) so identical runs are adjacent.
+func Compare(results []Result) Comparison {
+	cmp := Comparison{Runs: make([]RunRecord, 0, len(results))}
+	for _, r := range results {
+		rec := RunRecord{
+			Model: r.Model, Provider: r.Provider, Output: truncate(r.Output, 200),
+			Tokens: r.TokensUsed, CostUSD: r.CostUSD,
+			Duration: r.Duration.String(), Repro: r.ReproHash,
+		}
+		cmp.Runs = append(cmp.Runs, rec)
+	}
+	sort.SliceStable(cmp.Runs, func(i, j int) bool { return cmp.Runs[i].Repro < cmp.Runs[j].Repro })
+
+	unique := map[string]bool{}
+	for _, rec := range cmp.Runs {
+		if rec.Repro != "" {
+			unique[rec.Repro] = true
+		}
+		if cmp.MinTokens == 0 || rec.Tokens < cmp.MinTokens {
+			cmp.MinTokens = rec.Tokens
+		}
+		if rec.Tokens > cmp.MaxTokens {
+			cmp.MaxTokens = rec.Tokens
+		}
+		if cmp.MinCostUSD == 0 || rec.CostUSD < cmp.MinCostUSD {
+			cmp.MinCostUSD = rec.CostUSD
+		}
+		if rec.CostUSD > cmp.MaxCostUSD {
+			cmp.MaxCostUSD = rec.CostUSD
+		}
+	}
+	cmp.TotalRuns = len(cmp.Runs)
+	cmp.UniqueHashes = len(unique)
+	return cmp
+}
+
+// FormatComparison renders a human-readable comparative report.
+func FormatComparison(cmp Comparison) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Runs: %d (unique repro hashes: %d)\n", cmp.TotalRuns, cmp.UniqueHashes)
+	fmt.Fprintf(&b, "Tokens: %d..%d | Cost USD: %.6f..%.6f\n", cmp.MinTokens, cmp.MaxTokens, cmp.MinCostUSD, cmp.MaxCostUSD)
+	for _, rec := range cmp.Runs {
+		fmt.Fprintf(&b, "- %s/%s tokens=%d cost=%.6f dur=%s repro=%s\n", rec.Provider, rec.Model, rec.Tokens, rec.CostUSD, rec.Duration, shortHash(rec.Repro))
+	}
+	return strings.TrimRight(b.String(), "\n")
+}
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n]
+}
+
+func shortHash(h string) string {
+	if len(h) > 12 {
+		return h[:12]
+	}
+	return h
+}

internal/feature/evalloop/compare_test.go

@@ -0,0 +1,68 @@
+package evalloop
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestReproHashDeterministic(t *testing.T) {
+	in := Inputs{Model: "m", Provider: "p", Prompt: "task", ConfigVersion: 1}
+	tx := []byte("transcript")
+	a := ReproHashOf(in, tx)
+	b := ReproHashOf(in, tx)
+	if a != b {
+		t.Fatalf("hash must be deterministic, got %q vs %q", a, b)
+	}
+	if a == "" {
+		t.Fatal("hash must be non-empty")
+	}
+}
+
+func TestReproHashChangesOnInput(t *testing.T) {
+	in := Inputs{Model: "m", Provider: "p", Prompt: "task", ConfigVersion: 1}
+	base := ReproHashOf(in, []byte("tx"))
+	if ReproHashOf(in, []byte("other")) == base {
+		t.Fatal("hash must change when the transcript changes")
+	}
+	changed := in
+	changed.Prompt = "different"
+	if ReproHashOf(changed, []byte("tx")) == base {
+		t.Fatal("hash must change when inputs change")
+	}
+}
+
+func TestCompareAggregatesAndSorts(t *testing.T) {
+	results := []Result{
+		{Model: "m1", Provider: "p", TokensUsed: 100, CostUSD: 0.1, Duration: 1, ReproHash: "b"},
+		{Model: "m2", Provider: "p", TokensUsed: 200, CostUSD: 0.3, Duration: 2, ReproHash: "a"},
+	}
+	cmp := Compare(results)
+	if cmp.TotalRuns != 2 {
+		t.Fatalf("total runs = %d, want 2", cmp.TotalRuns)
+	}
+	if cmp.UniqueHashes != 2 {
+		t.Fatalf("unique hashes = %d, want 2", cmp.UniqueHashes)
+	}
+	if cmp.MinTokens != 100 || cmp.MaxTokens != 200 {
+		t.Fatalf("token range = %d..%d, want 100..200", cmp.MinTokens, cmp.MaxTokens)
+	}
+	if cmp.MinCostUSD != 0.1 || cmp.MaxCostUSD != 0.3 {
+		t.Fatalf("cost range = %f..%f, want 0.1..0.3", cmp.MinCostUSD, cmp.MaxCostUSD)
+	}
+	// Sorted by repro hash: "a" first.
+	if cmp.Runs[0].Model != "m2" {
+		t.Fatalf("first run = %s, want m2 (sorted by repro)", cmp.Runs[0].Model)
+	}
+}
+
+func TestFormatComparison(t *testing.T) {
+	cmp := Compare([]Result{
+		{Model: "m", Provider: "p", TokensUsed: 50, CostUSD: 0.05, Duration: 1, ReproHash: "abc123"},
+	})
+	out := FormatComparison(cmp)
+	for _, want := range []string{"Runs: 1", "unique repro hashes: 1", "Tokens: 50", "p/m"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("format missing %q: %s", want, out)
+		}
+	}
+}

internal/feature/evalloop/runtime.go

@@ -22,6 +22,9 @@ type Event struct {
 
 // Result is the outcome of one agent-runtime evaluation.
 type Result struct {
+	// Model and Provider identify the backend that produced this run.
+	Model    string `json:"model,omitempty"`
+	Provider string `json:"provider,omitempty"`
 	// Output is the concatenated assistant output produced by the loop.
 	Output string `json:"output"`
 	// Events are the normalized loop events in order.
@@ -34,6 +37,10 @@ type Result struct {
 	Transcript []byte `json:"-"`
 	// Duration is the wall-clock time of the run.
 	Duration time.Duration `json:"duration"`
+	// ReproHash is a deterministic SHA-256 over the run inputs (model,
+	// provider, prompt, config version) and the transcript, enabling identical
+	// runs to be compared or cached.
+	ReproHash string `json:"repro_hash,omitempty"`
 }
 
 // Runtime executes one agent-runtime evaluation.

internal/feature/evalloop/session.go

@@ -70,6 +70,8 @@ func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Resul
 		}
 	}
 	result.Duration = time.Since(start)
+	result.Model = r.Model
+	result.Provider = r.Provider
 
 	// Snapshot the transcript for offline replay of failing runs.
 	if msgs := sess.Persistence().RawMessages(); msgs != nil {
@@ -78,6 +80,12 @@ func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Resul
 		}
 	}
 
+	// Reproducibility hash over the fixed inputs and the transcript.
+	result.ReproHash = ReproHashOf(Inputs{
+		Model: r.Model, Provider: r.Provider, Prompt: prompt,
+		ConfigVersion: evalConfigVersion,
+	}, result.Transcript)
+
 	// Report usage/cost when the backend exposes it via the session cost model.
 	cost := sess.CostValue()
 	if cost != nil {
@@ -86,3 +94,7 @@ func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Resul
 	}
 	return result, nil
 }
+
+// evalConfigVersion is bumped whenever loop limits or system-prompt semantics
+// change, so reproducibility hashes are invalidated across versions.
+const evalConfigVersion = 1