From 5c271527ee05daaca466341597971b08ee64e9d8 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sat, 22 Aug 2026 21:34:31 +0530 Subject: [PATCH] feat(verify): adopt grok-cli incremental compaction and app-verify workflow Two adoptions from a deep-dive of superagent-ai/grok-cli, restricted to the capabilities verified as genuinely missing from hawk/eyrie: Incremental compaction (internal/engine/compact): - New BuildIncrementalCompactPrompt merges NEW messages into a persisted prior summary instead of re-summarizing the whole transcript; ExtractPriorSummary/PriorSummaryPrefix locate the prior [Conversation summary] block. - generateSummary now detects a prior summary and feeds only post-summary messages through the update prompt. Split-turn handling already existed; only the incremental path was missing. App verification workflow (internal/appverify + AppVerify tool): - Deterministic recipe detection (go/node/python/rust) covering the part ProjectVerify does not: start command, port, and smoke kind (http/cli/none), plus strict Normalize for untrusted recipe JSON. - Manifest contract at .hawk/verify/environment.json: LoadOrDetect persists detection so repeat runs are reproducible; corrupt manifests error instead of silently shadowing detection. - Phased QA prompt (Setup -> Build/Test -> Boot -> Evidence -> Teardown) encoding the discipline that build success is not proof: the app must boot and evidence artifacts are mandatory even on failure. - AppVerify tool (detect | manifest | smoke) with bounded readiness polling, fixed argv execution (no shell), guaranteed teardown, registered in cmd/chat_tools.go with safety capabilities and permission aliases. Remaining grok-cli candidates were assessed out of scope for now: media generation and provider Batch API need provider-side endpoints; X search and Telegram voice STT need provider support; desktop computer-use needs a native accessibility backend. Verification: go build ./... clean; new tests green (appverify 11, compact incremental 5, AppVerify tool 4); engine, compact, tool, safety, cmd and testaudit suites pass; golangci-lint 0 issues; gofmt clean. --- cmd/chat_tools.go | 1 + internal/appverify/appverify_test.go | 189 +++++++++++ internal/appverify/manifest.go | 82 +++++ internal/appverify/prompt.go | 93 ++++++ internal/appverify/recipe.go | 348 ++++++++++++++++++++ internal/engine/compact.go | 20 +- internal/engine/compact/incremental_test.go | 63 ++++ internal/engine/compact/prompt.go | 68 ++++ internal/engine/compact_reexports.go | 14 +- internal/engine/safety/capabilities.go | 1 + internal/engine/safety/permission.go | 2 + internal/tool/app_verify.go | 217 ++++++++++++ internal/tool/app_verify_test.go | 96 ++++++ 13 files changed, 1191 insertions(+), 3 deletions(-) create mode 100644 internal/appverify/appverify_test.go create mode 100644 internal/appverify/manifest.go create mode 100644 internal/appverify/prompt.go create mode 100644 internal/appverify/recipe.go create mode 100644 internal/engine/compact/incremental_test.go create mode 100644 internal/tool/app_verify.go create mode 100644 internal/tool/app_verify_test.go diff --git a/cmd/chat_tools.go b/cmd/chat_tools.go index a8781e77..0e6bc559 100644 --- a/cmd/chat_tools.go +++ b/cmd/chat_tools.go @@ -106,6 +106,7 @@ func optionalTools() []tool.Tool { tool.DebuggerTool{}, tool.DevEnvTool{}, tool.ProjectVerifyTool{}, + tool.AppVerifyTool{}, tool.DependencyAuditTool{}, tool.GitHubTool{}, &tool.PRGeneratorTool{}, diff --git a/internal/appverify/appverify_test.go b/internal/appverify/appverify_test.go new file mode 100644 index 00000000..0428df7c --- /dev/null +++ b/internal/appverify/appverify_test.go @@ -0,0 +1,189 @@ +package appverify + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeProject(t *testing.T, files map[string]string) string { + t.Helper() + root := t.TempDir() + for name, content := range files { + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +func TestDetectNextJS(t *testing.T) { + root := writeProject(t, map[string]string{ + "package.json": `{"name":"app","scripts":{"dev":"next dev","build":"next build","test":"jest"}, + "dependencies":{"next":"14.0.0","react":"18.0.0"}}`, + }) + r := Detect(root) + if r.Ecosystem != "node" || r.AppKind != "web" || r.Port != 3000 { + t.Fatalf("recipe = %+v", r) + } + if r.SmokeKind != SmokeHTTP || r.SmokeTarget() != "http://127.0.0.1:3000/" { + t.Fatalf("smoke = %q target %q", r.SmokeKind, r.SmokeTarget()) + } + want := []string{"npm", "run", "dev"} + if strings.Join(r.Start, " ") != strings.Join(want, " ") { + t.Fatalf("start = %v", r.Start) + } +} + +func TestDetectNodeLibraryNoSmoke(t *testing.T) { + root := writeProject(t, map[string]string{ + "package.json": `{"name":"lib","scripts":{"test":"vitest"},"devDependencies":{"vite":"5"}}`, + }) + r := Detect(root) + // vite is a web framework marker; a lib with only vite and no start script + // still has no boot path. + if r.Start != nil { + t.Fatalf("unexpected start %v", r.Start) + } + if r.SmokeKind == SmokeHTTP { + t.Fatalf("library must not claim http smoke: %+v", r) + } +} + +func TestDetectGoCLIAndLibrary(t *testing.T) { + cliRoot := writeProject(t, map[string]string{"go.mod": "module x\n\ngo 1.22\n", "main.go": "package main\n"}) + r := Detect(cliRoot) + if r.Ecosystem != "go" || r.AppKind != "cli" || r.SmokeKind != SmokeCLI { + t.Fatalf("cli recipe = %+v", r) + } + libRoot := writeProject(t, map[string]string{"go.mod": "module y\n\ngo 1.22\n"}) + r = Detect(libRoot) + if r.AppKind != "library" || r.SmokeKind != SmokeNone { + t.Fatalf("lib recipe = %+v", r) + } +} + +func TestDetectDjango(t *testing.T) { + root := writeProject(t, map[string]string{ + "manage.py": "#!/usr/bin/env python\n", + "requirements.txt": "django\n", + }) + r := Detect(root) + if r.Ecosystem != "python" || r.AppLabel != "django app" || r.Port != 8000 || r.SmokeKind != SmokeHTTP { + t.Fatalf("recipe = %+v", r) + } +} + +func TestDetectRust(t *testing.T) { + root := writeProject(t, map[string]string{"Cargo.toml": "[package]\nname=\"x\"\n"}) + r := Detect(root) + if r.Ecosystem != "rust" || len(r.Test) == 0 || r.SmokeKind != SmokeCLI { + t.Fatalf("recipe = %+v", r) + } +} + +func TestDetectUnknown(t *testing.T) { + r := Detect(t.TempDir()) + if r.Ecosystem != "unknown" || r.SmokeKind != SmokeNone || len(r.Notes) == 0 { + t.Fatalf("recipe = %+v", r) + } +} + +func TestManifestRoundTripAndPriority(t *testing.T) { + root := t.TempDir() + r, existed, err := LoadOrDetect(root) + if err != nil || existed { + t.Fatalf("LoadOrDetect = (%+v,%v,%v)", r, existed, err) + } + if _, err := os.Stat(ManifestPath(root)); err != nil { + t.Fatalf("manifest not persisted: %v", err) + } + // Second load reads the manifest, not detection. + again, existed2, err := LoadOrDetect(root) + if err != nil || !existed2 { + t.Fatalf("second LoadOrDetect = (%+v,%v,%v)", again, existed2, err) + } + if again.Ecosystem != r.Ecosystem { + t.Fatalf("round trip mismatch") + } +} + +func TestLoadManifestCorruptIsError(t *testing.T) { + root := writeProject(t, map[string]string{ + ".hawk/verify/environment.json": "{not json", + }) + if _, err := LoadManifest(root); err == nil { + t.Fatal("expected error for corrupt manifest") + } +} + +func TestNormalizeFiltersGarbage(t *testing.T) { + raw, _ := json.Marshal(map[string]interface{}{ + "ecosystem": " Node! ", + "appKind": "WEB", + "port": 99999, + "smokeKind": "bogus", + "install": []string{"npm", "", "ci\ninjected", "ci"}, + "start": []string{"npm", "start"}, + }) + r, err := Normalize(raw) + if err != nil { + t.Fatalf("Normalize: %v", err) + } + if r.Ecosystem != "node" || r.AppKind != "web" { + t.Fatalf("tokens not sanitized: %+v", r) + } + if r.Port != 0 { + t.Fatalf("out-of-range port kept: %d", r.Port) + } + if r.SmokeKind != SmokeNone { + t.Fatalf("bogus smoke kind not reset: %q", r.SmokeKind) + } + for _, a := range r.Install { + if strings.ContainsAny(a, "\n\r\x00") { + t.Fatalf("unsafe arg survived: %q", a) + } + } + if got := strings.Join(r.Install, " "); got != "npm ci" { + t.Fatalf("install = %q", got) + } +} + +func TestNormalizeHTTPRequiresStart(t *testing.T) { + raw, _ := json.Marshal(map[string]interface{}{"ecosystem": "go", "smokeKind": "http", "port": 8080}) + r, err := Normalize(raw) + if err != nil { + t.Fatal(err) + } + if r.SmokeKind != SmokeNone { + t.Fatalf("http without start must downgrade to none, got %q", r.SmokeKind) + } +} + +func TestBuildVerifyPromptPhases(t *testing.T) { + r := Recipe{ + Ecosystem: "node", AppKind: "web", AppLabel: "next.js app", + Install: []string{"npm", "ci"}, + Build: []string{"npm", "run", "build"}, + Test: []string{"npm", "test"}, + Start: []string{"npm", "run", "dev"}, + Port: 3000, + SmokeKind: SmokeHTTP, + } + p := BuildVerifyPrompt(r) + for _, want := range []string{ + "Phase 1 — Setup", "Phase 2 — Build and test", "Phase 3 — Boot the app", + "Phase 4 — Evidence", "Phase 5 — Teardown", + "http://127.0.0.1:3000/", EvidenceDir, "NOT success", + } { + if !strings.Contains(p, want) { + t.Fatalf("prompt missing %q", want) + } + } +} diff --git a/internal/appverify/manifest.go b/internal/appverify/manifest.go new file mode 100644 index 00000000..576be01b --- /dev/null +++ b/internal/appverify/manifest.go @@ -0,0 +1,82 @@ +package appverify + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/GrayCodeAI/hawk/internal/safewrite" +) + +// ManifestPath returns the location of the persisted verify environment +// manifest for a project: /.hawk/verify/environment.json. +func ManifestPath(root string) string { + return filepath.Join(root, ".hawk", "verify", "environment.json") +} + +// Manifest is the on-disk contract between recipe detection and execution. +// Once written, it is the highest-priority source of truth for verification: +// deterministic detection only runs when no manifest exists yet. +type Manifest struct { + Recipe Recipe `json:"recipe"` +} + +// LoadManifest reads the manifest from disk. os.ErrNotExist is returned when +// no manifest has been persisted, so callers can fall back to Detect. +func LoadManifest(root string) (Recipe, error) { + raw, err := os.ReadFile(ManifestPath(root)) // #nosec G304 -- path built from caller-provided root + if err != nil { + return Recipe{}, err + } + var m Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return Recipe{}, fmt.Errorf("appverify: parse manifest: %w", err) + } + // Re-validate through Normalize so a hand-edited or agent-written manifest + // cannot smuggle unsafe values. + canonical, err := json.Marshal(m.Recipe) + if err != nil { + return Recipe{}, err + } + return Normalize(canonical) +} + +// SaveManifest persists the recipe as the project's verify manifest. Existing +// manifests are overwritten atomically. +func SaveManifest(root string, r Recipe) error { + if r.Ecosystem == "" { + return fmt.Errorf("appverify: refusing to save recipe without ecosystem") + } + data, err := json.MarshalIndent(Manifest{Recipe: r}, "", " ") + if err != nil { + return fmt.Errorf("appverify: encode manifest: %w", err) + } + path := ManifestPath(root) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("appverify: create manifest dir: %w", err) + } + if err := safewrite.WriteFile(path, append(data, '\n')); err != nil { + return fmt.Errorf("appverify: write manifest: %w", err) + } + return nil +} + +// LoadOrDetect returns the persisted manifest recipe when present, otherwise +// runs deterministic detection and persists the result so subsequent runs are +// reproducible. The second return value reports whether the manifest already +// existed (false means it was just created from detection). +func LoadOrDetect(root string) (Recipe, bool, error) { + if r, err := LoadManifest(root); err == nil { + return r, true, nil + } else if !os.IsNotExist(err) { + // A corrupt manifest must not silently shadow detection; surface it. + return Recipe{}, false, fmt.Errorf("appverify: existing manifest invalid: %w", err) + } + r := Detect(root) + if err := SaveManifest(root, r); err != nil { + // Detection still works without persistence; report the recipe anyway. + return r, false, nil + } + return r, false, nil +} diff --git a/internal/appverify/prompt.go b/internal/appverify/prompt.go new file mode 100644 index 00000000..19b17fc5 --- /dev/null +++ b/internal/appverify/prompt.go @@ -0,0 +1,93 @@ +package appverify + +import ( + "fmt" + "strings" +) + +// EvidenceDir is the stable, workspace-relative directory for verification +// artifacts. Stable paths let reports and downstream tooling rely on them. +const EvidenceDir = ".hawk/verify/artifacts" + +// BuildVerifyPrompt renders the phased QA-engineer prompt for the recipe. The +// discipline it encodes (adopted from grok-cli) is that build/test passing +// means nothing unless the app actually boots and serves: the workflow is +// mandatory and evidence is mandatory even on failure. +func BuildVerifyPrompt(r Recipe) string { + var b strings.Builder + b.WriteString("You are verifying this project end-to-end as a QA engineer. ") + b.WriteString("A green build is NOT success — the app must actually run. Follow every phase in order.\n\n") + + fmt.Fprintf(&b, "Project: %s (%s/%s)\n", r.AppLabel, r.Ecosystem, r.AppKind) + if len(r.Notes) > 0 { + b.WriteString("\nRecipe notes:\n") + for _, n := range r.Notes { + fmt.Fprintf(&b, "- %s\n", n) + } + } + + b.WriteString(` +## Phase 1 — Setup +- Probe for required runtimes before installing anything; only install what is missing. +`) + + if len(r.Install) > 0 { + fmt.Fprintf(&b, "- Install dependencies with exactly: %s\n", argv(r.Install)) + } + + b.WriteString(` +## Phase 2 — Build and test +`) + if len(r.Build) > 0 { + fmt.Fprintf(&b, "- Build: %s\n", argv(r.Build)) + } + if len(r.Test) > 0 { + fmt.Fprintf(&b, "- Test: %s\n", argv(r.Test)) + } + if len(r.Build) == 0 && len(r.Test) == 0 { + b.WriteString("- No build/test commands in the recipe; record that plainly.\n") + } + + b.WriteString(` +## Phase 3 — Boot the app (REQUIRED) +`) + switch { + case len(r.Start) > 0 && r.SmokeKind == SmokeHTTP: + target := r.SmokeTarget() + fmt.Fprintf(&b, "- Start the app in the background: %s\n", argv(r.Start)) + fmt.Fprintf(&b, "- Wait for readiness by polling %s until HTTP 200 (bounded loop, ~60s max).\n", target) + b.WriteString("- If readiness never succeeds, capture the app log tail and report the exact command used.\n") + case len(r.Start) > 0 && r.SmokeKind == SmokeCLI: + fmt.Fprintf(&b, "- Run the entrypoint once: %s\n", argv(r.Start)) + b.WriteString("- Treat a zero exit code plus sane stdout as the boot signal.\n") + default: + b.WriteString("- No start command is known. Inspect the project to determine how it runs; if it cannot be booted, say so explicitly instead of guessing.\n") + } + + b.WriteString(` +## Phase 4 — Evidence (REQUIRED even on failure) +`) + fmt.Fprintf(&b, "- Save all artifacts under %s:\n", EvidenceDir) + b.WriteString(" - app log tail -> artifacts/app.log\n") + if r.SmokeKind == SmokeHTTP { + b.WriteString(" - screenshot of the served page when a browser/screenshot tool is available\n") + } + b.WriteString(`- Report in this exact structure: + Summary / Results / Evidence (exact artifact paths) / Blockers / Residual Risk. + +## Phase 5 — Teardown +- Stop any background process you started BEFORE finishing, then verify no orphan listeners remain on the port. +`) + + if r.Ecosystem == "node" { + b.WriteString(` +Bounded retry guidance: +- Native module build failures (lightningcss, sharp, @next/swc, esbuild): remove node_modules and reinstall once with optional deps enabled, then rebuild. At most one retry. +- Startup readiness failures: retry at most once binding HOST=0.0.0.0 and the explicit PORT. +- Anything else: do not thrash — report the blocker directly. +`) + } + return b.String() +} + +func argv(args []string) string { return "`" + strings.Join(args, " ") + "`" } diff --git a/internal/appverify/recipe.go b/internal/appverify/recipe.go new file mode 100644 index 00000000..b9563d9e --- /dev/null +++ b/internal/appverify/recipe.go @@ -0,0 +1,348 @@ +// Package appverify implements the "prove it works" project-verification +// workflow adopted from grok-cli's verify subsystem: a deterministic recipe +// (how to install, build, test, start, and smoke-check the app), a persisted +// manifest that acts as the contract between static detection and agentic +// execution, and the phased QA prompt that turns "it builds" into "it boots, +// serves, and shows evidence". +package appverify + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// SmokeKind describes how an app's liveness is proven after boot. +type SmokeKind string + +const ( + // SmokeHTTP proves readiness by polling an HTTP endpoint. + SmokeHTTP SmokeKind = "http" + // SmokeCLI apps are one-shot commands with no server to poll. + SmokeCLI SmokeKind = "cli" + // SmokeNone means no smoke check is available. + SmokeNone SmokeKind = "none" +) + +// Recipe is the deterministic description of how to verify an app. Fixed +// argument lists (no shell strings) keep execution bounded and safe. +type Recipe struct { + Ecosystem string `json:"ecosystem"` // go, node, python, rust, java, unknown + AppKind string `json:"appKind"` // web, api, cli, library + AppLabel string `json:"appLabel"` // human-readable, e.g. "nextjs app" + Install []string `json:"install,omitempty"` // fixed argv, e.g. ["npm","ci"] + Build []string `json:"build,omitempty"` // fixed argv + Test []string `json:"test,omitempty"` // fixed argv + Start []string `json:"start,omitempty"` // fixed argv for booting the app + Port int `json:"port,omitempty"` // expected listen port when known + SmokeKind SmokeKind `json:"smokeKind"` // http | cli | none + Evidence []string `json:"evidence,omitempty"` // artifact paths produced by verification + Notes []string `json:"notes,omitempty"` // why this recipe was chosen / caveats +} + +// Detect infers a Recipe deterministically from project markers in root. It +// always returns a usable recipe; unknown projects get smokeKind "none" plus a +// note telling the caller to inspect directly. +func Detect(root string) Recipe { + if r, ok := detectNode(root); ok { + return r + } + if r, ok := detectGo(root); ok { + return r + } + if r, ok := detectPython(root); ok { + return r + } + if r, ok := detectRust(root); ok { + return r + } + return Recipe{ + Ecosystem: "unknown", + AppKind: "unknown", + AppLabel: "unrecognized project", + SmokeKind: SmokeNone, + Notes: []string{"no recognized project markers — inspect the tree directly"}, + } +} + +func hasFile(root, name string) bool { + st, err := os.Stat(filepath.Join(root, name)) + return err == nil && !st.IsDir() +} + +// node framework defaults: dependency marker → (kind, default dev port). +var nodeFrameworks = []struct { + dependency string + appKind string + label string + port int +}{ + {"next", "web", "next.js app", 3000}, + {"@sveltejs/kit", "web", "sveltekit app", 5173}, + {"astro", "web", "astro app", 4321}, + {"@remix-run/react", "web", "remix app", 3000}, + {"react-scripts", "web", "create-react-app", 3000}, + {"vite", "web", "vite app", 5173}, + {"express", "api", "express api", 3000}, + {"fastify", "api", "fastify api", 3000}, +} + +type packageJSON struct { + Name string `json:"name"` + Scripts map[string]string `json:"scripts"` + Dependencies map[string]string `json:"dependencies"` + DevDeps map[string]string `json:"devDependencies"` +} + +// pickPackageManager chooses the runner from lockfile presence. +func pickPackageManager(root string) string { + switch { + case hasFile(root, "pnpm-lock.yaml"): + return "pnpm" + case hasFile(root, "bun.lockb"), hasFile(root, "bun.lock"): + return "bun" + case hasFile(root, "yarn.lock"): + return "yarn" + default: + return "npm" + } +} + +// runScript builds the fixed argv for running a package.json script under the +// detected package manager. +func runScript(manager, script string) []string { + switch manager { + case "pnpm": + return []string{"pnpm", "run", script} + case "bun": + return []string{"bun", "run", script} + case "yarn": + return []string{"yarn", script} + default: + return []string{"npm", "run", script} + } +} + +func detectNode(root string) (Recipe, bool) { + raw, err := os.ReadFile(filepath.Join(root, "package.json")) // #nosec G304 -- path built from caller-provided root + if err != nil { + return Recipe{}, false + } + var pkg packageJSON + if err := json.Unmarshal(raw, &pkg); err != nil { + return Recipe{}, false + } + + manager := pickPackageManager(root) + deps := map[string]string{} + for k, v := range pkg.Dependencies { + deps[k] = v + } + for k, v := range pkg.DevDeps { + deps[k] = v + } + + r := Recipe{Ecosystem: "node"} + for _, fw := range nodeFrameworks { + if _, ok := deps[fw.dependency]; ok { + r.AppKind = fw.appKind + r.AppLabel = fw.label + r.Port = fw.port + break + } + } + if r.AppKind == "" { + r.AppKind = "library" + r.AppLabel = "node package" + } + + // Prefer explicit dev/start scripts for booting. + switch { + case pkg.Scripts["dev"] != "": + r.Start = runScript(manager, "dev") + case pkg.Scripts["start"] != "": + r.Start = runScript(manager, "start") + } + + if hasFile(root, "package-lock.json") || hasFile(root, "pnpm-lock.yaml") || + hasFile(root, "yarn.lock") || hasFile(root, "bun.lockb") || hasFile(root, "bun.lock") { + switch manager { + case "pnpm": + r.Install = []string{"pnpm", "install", "--frozen-lockfile"} + case "bun": + r.Install = []string{"bun", "install", "--frozen-lockfile"} + case "yarn": + r.Install = []string{"yarn", "install", "--frozen-lockfile"} + default: + r.Install = []string{"npm", "ci"} + } + } else if _, ok := deps["next"]; ok || len(deps) > 0 { + r.Install = runScript(manager, "install") + } + + if pkg.Scripts["build"] != "" { + r.Build = runScript(manager, "build") + } + if pkg.Scripts["test"] != "" { + r.Test = runScript(manager, "test") + } + + classifySmoke(&r) + if r.SmokeKind == SmokeNone && r.Start != nil { + r.Notes = append(r.Notes, "start script present but no port inferred; confirm the listen port before HTTP smoke checks") + } + return r, true +} + +func detectGo(root string) (Recipe, bool) { + if !hasFile(root, "go.mod") { + return Recipe{}, false + } + r := Recipe{ + Ecosystem: "go", + Build: []string{"go", "build", "./..."}, + Test: []string{"go", "test", "./..."}, + } + if hasFile(root, "main.go") { + r.AppKind = "cli" + r.AppLabel = "go application" + r.Start = []string{"go", "run", "."} + r.SmokeKind = SmokeCLI + } else { + r.AppKind = "library" + r.AppLabel = "go module" + r.SmokeKind = SmokeNone + } + return r, true +} + +func detectPython(root string) (Recipe, bool) { + isPy := hasFile(root, "pyproject.toml") || hasFile(root, "requirements.txt") || + hasFile(root, "manage.py") || hasFile(root, "setup.py") + if !isPy { + return Recipe{}, false + } + r := Recipe{Ecosystem: "python"} + switch { + case hasFile(root, "manage.py"): + r.AppKind = "web" + r.AppLabel = "django app" + r.Install = []string{"python3", "-m", "pip", "install", "-r", "requirements.txt"} + r.Test = []string{"python3", "manage.py", "test"} + r.Start = []string{"python3", "manage.py", "runserver", "0.0.0.0:8000"} + r.Port = 8000 + default: + r.AppKind = "library" + r.AppLabel = "python project" + if hasFile(root, "requirements.txt") { + r.Install = []string{"python3", "-m", "pip", "install", "-r", "requirements.txt"} + } + r.Test = []string{"python3", "-m", "pytest"} + } + classifySmoke(&r) + return r, true +} + +func detectRust(root string) (Recipe, bool) { + if !hasFile(root, "Cargo.toml") { + return Recipe{}, false + } + r := Recipe{ + Ecosystem: "rust", + AppKind: "cli", + AppLabel: "rust binary", + Install: nil, // cargo fetches on build + Build: []string{"cargo", "build"}, + Test: []string{"cargo", "test"}, + Start: []string{"cargo", "run"}, + SmokeKind: SmokeCLI, + } + return r, true +} + +// classifySmoke sets SmokeKind/Port based on what the rest of the recipe +// implies: a Start command with a known port means HTTP smoke is possible. +func classifySmoke(r *Recipe) { + if r.Port > 0 && r.Start != nil { + r.SmokeKind = SmokeHTTP + return + } + if r.Start != nil && r.SmokeKind == "" { + r.SmokeKind = SmokeCLI + return + } + if r.SmokeKind == "" { + r.SmokeKind = SmokeNone + } +} + +// Normalize validates and canonicalizes a recipe parsed from untrusted JSON +// (e.g. an LLM-proposed manifest). It filters garbage rather than failing so +// callers can safely round-trip agent output. +func Normalize(raw []byte) (Recipe, error) { + var r Recipe + if err := json.Unmarshal(raw, &r); err != nil { + return Recipe{}, fmt.Errorf("appverify: parse recipe: %w", err) + } + r.Ecosystem = sanitizeToken(r.Ecosystem) + r.AppKind = sanitizeToken(r.AppKind) + r.AppLabel = strings.TrimSpace(r.AppLabel) + if r.Ecosystem == "" { + return Recipe{}, fmt.Errorf("appverify: recipe missing ecosystem") + } + switch r.SmokeKind { + case SmokeHTTP, SmokeCLI, SmokeNone: + case "": + r.SmokeKind = SmokeNone + default: + r.SmokeKind = SmokeNone + } + if r.Port < 0 || r.Port > 65535 { + r.Port = 0 + } + r.Install = sanitizeArgs(r.Install) + r.Build = sanitizeArgs(r.Build) + r.Test = sanitizeArgs(r.Test) + r.Start = sanitizeArgs(r.Start) + if r.Start == nil && r.SmokeKind == SmokeHTTP { + r.SmokeKind = SmokeNone + } + return r, nil +} + +func sanitizeToken(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + for _, c := range s { + if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' { + b.WriteRune(c) + } + } + return b.String() +} + +// sanitizeArgs keeps only non-empty, control-character-free arguments. +func sanitizeArgs(args []string) []string { + out := make([]string, 0, len(args)) + for _, a := range args { + a = strings.TrimSpace(a) + if a == "" || strings.ContainsAny(a, "\x00\n\r") { + continue + } + out = append(out, a) + } + if len(out) == 0 { + return nil + } + return out +} + +// SmokeTarget renders the URL used for HTTP readiness polling. +func (r Recipe) SmokeTarget() string { + if r.Port <= 0 { + return "" + } + return fmt.Sprintf("http://127.0.0.1:%d/", r.Port) +} diff --git a/internal/engine/compact.go b/internal/engine/compact.go index 87aff5a2..adb59762 100644 --- a/internal/engine/compact.go +++ b/internal/engine/compact.go @@ -121,10 +121,26 @@ func (s *Session) smartCompactBody(ctx context.Context) { const summaryInputRuneCap = 32_000 func (s *Session) generateSummary(ctx context.Context, raw []types.EyrieMessage) string { + // Incremental compaction: if a prior summary was persisted by an earlier + // compaction, merge the NEW messages into it rather than re-summarizing the + // entire conversation from scratch. This preserves already-captured context + // and avoids the cost of re-deriving it. + prior := ExtractPriorSummary(raw) + newMsgs := raw + if prior != "" { + // Only the messages that came after the persisted summary are "new". + for i, m := range raw { + if m.Role == "user" && strings.HasPrefix(m.Content, PriorSummaryPrefix) { + newMsgs = raw[i+1:] + break + } + } + } + // Build a compact version of the conversation for summarization // using the structured compaction prompt from compact_prompt.go var summaryMsgs []types.EyrieMessage - compactPrompt := BuildCompactPrompt(CompactBase) + compactPrompt := BuildIncrementalCompactPrompt(prior) summaryMsgs = append(summaryMsgs, types.EyrieMessage{ Role: "user", Content: compactPrompt + "\n\nConversation:\n", @@ -132,7 +148,7 @@ func (s *Session) generateSummary(ctx context.Context, raw []types.EyrieMessage) // Add a condensed version of messages, capped at a bounded total size budget := summaryInputRuneCap - for _, m := range raw { + for _, m := range newMsgs { if m.Role != "user" && m.Role != "assistant" { continue } diff --git a/internal/engine/compact/incremental_test.go b/internal/engine/compact/incremental_test.go new file mode 100644 index 00000000..ffb89fe2 --- /dev/null +++ b/internal/engine/compact/incremental_test.go @@ -0,0 +1,63 @@ +package compact + +import ( + "strings" + "testing" + + "github.com/GrayCodeAI/hawk/internal/types" +) + +func TestBuildIncrementalCompactPromptIncludesPriorSummary(t *testing.T) { + prior := "## Goal\n- build auth\n\n## Progress\n### Done\n- x" + p := BuildIncrementalCompactPrompt(prior) + if !strings.Contains(p, prior) { + t.Fatal("incremental prompt must embed the prior summary") + } + if !strings.Contains(p, "") { + t.Fatal("incremental prompt must use the previous-summary tags") + } + if !strings.Contains(p, "## Progress") { + t.Fatal("incremental prompt must preserve the section structure") + } +} + +func TestBuildIncrementalCompactPromptEmptyPriorFallsBack(t *testing.T) { + p := BuildIncrementalCompactPrompt("") + if strings.Contains(p, "") { + t.Fatal("empty prior summary should fall back to the base prompt") + } + if !strings.Contains(p, "## Next Step") { + t.Fatal("fallback prompt must be the full structured summary template") + } +} + +func TestExtractPriorSummaryFound(t *testing.T) { + msgs := []types.EyrieMessage{ + {Role: "user", Content: "[Conversation summary]\n## Goal\n- build\n\n[Continue from the recent messages below.]"}, + {Role: "assistant", Content: "ok"}, + } + got := ExtractPriorSummary(msgs) + if !strings.Contains(got, "## Goal") || strings.Contains(got, "[Continue") { + t.Fatalf("ExtractPriorSummary = %q", got) + } +} + +func TestExtractPriorSummaryNone(t *testing.T) { + msgs := []types.EyrieMessage{ + {Role: "user", Content: "hello"}, + } + if got := ExtractPriorSummary(msgs); got != "" { + t.Fatalf("expected empty, got %q", got) + } +} + +func TestExtractPriorSummaryIgnoresNonSummaryFirst(t *testing.T) { + msgs := []types.EyrieMessage{ + {Role: "user", Content: "[Session memory summary]\nstuff"}, + {Role: "user", Content: "[Conversation summary]\n## Goal\n- x"}, + } + got := ExtractPriorSummary(msgs) + if !strings.Contains(got, "## Goal") { + t.Fatalf("ExtractPriorSummary should find the conversation summary, got %q", got) + } +} diff --git a/internal/engine/compact/prompt.go b/internal/engine/compact/prompt.go index 82e49672..6a5eefcf 100644 --- a/internal/engine/compact/prompt.go +++ b/internal/engine/compact/prompt.go @@ -1,5 +1,12 @@ package compact +import ( + "fmt" + "strings" + + "github.com/GrayCodeAI/hawk/internal/types" +) + const noToolsPreamble = `CRITICAL: Respond with TEXT ONLY. Do NOT call any tools. - Do NOT use Read, Bash, Grep, Glob, Edit, Write, or ANY other tool. @@ -67,6 +74,31 @@ const summaryTemplate = `Now provide your summary inside tags using EX ## Next Step - [based on most recent user messages, what should happen next — include direct quotes if user gave specific direction]` +// incrementalUpdateTemplate instructs the model to merge the new conversation +// into a prior summary rather than re-summarizing from scratch. This preserves +// previously-captured context and avoids the cost of re-deriving it, while +// folding in only the progress made since the last compaction. +const incrementalUpdateTemplate = `A previous summary of this conversation exists (shown below in tags). + +Update that summary in place to reflect the NEW messages that were added after it. Do NOT re-derive facts already captured. Follow these rules: + +1. Preserve the existing sections and their structure EXACTLY (## Goal, ## Constraints & Preferences, ## Progress, ## Files Modified, ## Key Decisions, ## Errors & Fixes, ## User Instructions, ## Next Step). +2. Update each section only where the new messages add information: + - ## Goal: keep unless the user redefined the goal. + - ## Progress: add new Done/In Progress/Blocked entries; keep existing ones. + - ## Files Modified: add any newly read/created/modified files. + - ## Key Decisions: add new decisions; keep prior ones. + - ## Errors & Fixes: add newly encountered errors; keep prior ones. + - ## User Instructions (verbatim): append any new non-trivial user directions. + - ## Next Step: replace with the most recent next step. +3. If the new messages add no information to a section, leave it unchanged (do not empty it). + +Provide the fully updated summary inside tags using EXACTLY this structure. Keep section order unchanged. + + +%s +` + func BuildCompactPrompt(variant CompactVariant) string { var analysis string switch variant { @@ -78,6 +110,42 @@ func BuildCompactPrompt(variant CompactVariant) string { return noToolsPreamble + analysis + "\n\n" + summaryTemplate } +// BuildIncrementalCompactPrompt builds a prompt that merges the new +// conversation into an existing prior summary instead of re-summarizing from +// scratch. priorSummary is the previously generated structured summary. +func BuildIncrementalCompactPrompt(priorSummary string) string { + if priorSummary == "" { + return BuildCompactPrompt(CompactBase) + } + return noToolsPreamble + detailedAnalysisPartial + "\n\n" + + fmt.Sprintf(incrementalUpdateTemplate, priorSummary) +} + +// PriorSummaryPrefix is the marker prefix hawk prepends to a persisted +// conversation summary message. +const PriorSummaryPrefix = "[Conversation summary]" + +// ExtractPriorSummary extracts the previously generated summary text from the +// first message of a conversation if one was persisted by an earlier +// compaction. It returns "" when no prior summary is present. +func ExtractPriorSummary(msgs []types.EyrieMessage) string { + for _, m := range msgs { + if m.Role != "user" { + continue + } + if strings.HasPrefix(m.Content, PriorSummaryPrefix) { + body := strings.TrimPrefix(m.Content, PriorSummaryPrefix) + body = strings.TrimSpace(body) + // Stop at the continuation marker that follows the summary body. + if idx := strings.Index(body, "[Continue from the recent messages below.]"); idx >= 0 { + body = body[:idx] + } + return strings.TrimSpace(body) + } + } + return "" +} + type CompactVariant int const ( diff --git a/internal/engine/compact_reexports.go b/internal/engine/compact_reexports.go index a11ec4ee..e964ba1b 100644 --- a/internal/engine/compact_reexports.go +++ b/internal/engine/compact_reexports.go @@ -38,7 +38,19 @@ func NewCompactionTrigger(windowSize int) *CompactionTrigger { func BuildCompactPrompt(variant CompactVariant) string { return compact.BuildCompactPrompt(variant) } func FormatCompactSummary(raw string) string { return compact.FormatCompactSummary(raw) } -func IsCompactableTool(name string) bool { return compact.IsCompactableTool(name) } +func BuildIncrementalCompactPrompt(priorSummary string) string { + return compact.BuildIncrementalCompactPrompt(priorSummary) +} + +func ExtractPriorSummary(msgs []types.EyrieMessage) string { + return compact.ExtractPriorSummary(msgs) +} + +// PriorSummaryPrefix is the marker prefix hawk prepends to a persisted +// conversation summary message. +const PriorSummaryPrefix = compact.PriorSummaryPrefix + +func IsCompactableTool(name string) bool { return compact.IsCompactableTool(name) } func AdjustIndexToPreserveAPIInvariants(msgs []types.EyrieMessage, startIdx int) int { return compact.AdjustIndexToPreserveAPIInvariants(msgs, startIdx) } diff --git a/internal/engine/safety/capabilities.go b/internal/engine/safety/capabilities.go index fed223ac..ea9f1a69 100644 --- a/internal/engine/safety/capabilities.go +++ b/internal/engine/safety/capabilities.go @@ -51,6 +51,7 @@ var toolPolicies = map[string]ToolPolicy{ "GitHistory": {Name: "GitHistory", Capabilities: []Capability{CapabilityFilesystemRead, CapabilityProcessExecute}, DefaultRisk: RiskLow}, "Diagnostics": {Name: "Diagnostics", Capabilities: []Capability{CapabilityFilesystemRead, CapabilityProcessExecute}, DefaultRisk: RiskMedium}, "ProjectVerify": {Name: "ProjectVerify", Capabilities: []Capability{CapabilityFilesystemRead, CapabilityProcessExecute}, DefaultRisk: RiskMedium}, + "AppVerify": {Name: "AppVerify", Capabilities: []Capability{CapabilityFilesystemRead, CapabilityProcessExecute}, DefaultRisk: RiskMedium}, "DependencyAudit": {Name: "DependencyAudit", Capabilities: []Capability{CapabilityFilesystemRead, CapabilityProcessExecute, CapabilityNetworkAccess}, DefaultRisk: RiskMedium}, "Git": {Name: "Git", Capabilities: []Capability{CapabilityProcessExecute}, DefaultRisk: RiskMedium}, "GitHub": {Name: "GitHub", Capabilities: []Capability{CapabilityNetworkAccess, CapabilityProcessExecute}, DefaultRisk: RiskMedium}, diff --git a/internal/engine/safety/permission.go b/internal/engine/safety/permission.go index 3e8aefee..8942e1b5 100644 --- a/internal/engine/safety/permission.go +++ b/internal/engine/safety/permission.go @@ -319,6 +319,8 @@ func canonicalToolName(name string) string { return "ToolHealth" case "project_verify", "projectverify", "verify_project": return "ProjectVerify" + case "app_verify", "appverify", "verify_app": + return "AppVerify" case "dependency_audit", "dependencyaudit", "deps": return "DependencyAudit" case "git_history", "githistory", "git-history": diff --git a/internal/tool/app_verify.go b/internal/tool/app_verify.go new file mode 100644 index 00000000..f99b8c71 --- /dev/null +++ b/internal/tool/app_verify.go @@ -0,0 +1,217 @@ +package tool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "time" + + "github.com/GrayCodeAI/hawk/internal/appverify" +) + +// AppVerifyTool implements the "prove it works" workflow: detect a recipe, +// persist it as the project manifest contract, and boot-smoke the app with +// bounded readiness polling. It complements ProjectVerify (build/test/lint) +// by covering the part that actually proves the app runs. +type AppVerifyTool struct{} + +func (AppVerifyTool) Name() string { return "AppVerify" } +func (AppVerifyTool) RiskLevel() string { return "medium" } +func (AppVerifyTool) Aliases() []string { return []string{"app-verify", "verify_app"} } +func (AppVerifyTool) Description() string { + return "Detect how this project boots and prove it runs: infer an install/build/test/start recipe, persist it to .hawk/verify/environment.json as the verification contract, and run a bounded boot smoke check with readiness polling. Use action=detect to inspect, action=manifest to write/update the contract, action=smoke to boot the app and verify readiness." +} + +func (AppVerifyTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action": map[string]interface{}{ + "type": "string", + "enum": []string{"detect", "manifest", "smoke"}, + "description": "detect infers the recipe; manifest loads-or-detects and persists .hawk/verify/environment.json; smoke boots the app using the recipe's start command and polls readiness.", + }, + "path": map[string]interface{}{ + "type": "string", + "description": "Project directory (default: session working directory).", + }, + "readiness_seconds": map[string]interface{}{ + "type": "integer", + "minimum": 1, + "maximum": 300, + "description": "Max seconds to wait for the app to become ready during smoke (default 60).", + }, + }, + "required": []string{"action"}, + } +} + +type smokeResult struct { + Status string `json:"status"` // passed | failed | skipped + StartCmd string `json:"start_command,omitempty"` + SmokeKind string `json:"smoke_kind"` + Target string `json:"smoke_target,omitempty"` + Duration string `json:"duration,omitempty"` + Error string `json:"error,omitempty"` +} + +func (AppVerifyTool) Execute(ctx context.Context, input json.RawMessage) (string, error) { + var params struct { + Action string `json:"action"` + Path string `json:"path"` + ReadinessSeconds int `json:"readiness_seconds"` + } + if err := json.Unmarshal(input, ¶ms); err != nil { + return "", fmt.Errorf("invalid input: %w", err) + } + switch params.Action { + case "detect", "manifest", "smoke": + default: + return "", fmt.Errorf("unsupported action %q (use detect, manifest, or smoke)", params.Action) + } + + root := params.Path + if root == "" { + if tc := GetToolContext(ctx); tc != nil && tc.WorkingDir != "" { + root = tc.WorkingDir + } else { + var err error + root, err = os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve working directory: %w", err) + } + } + } + if err := validatePathAllowed(ctx, root); err != nil { + return "", err + } + + switch params.Action { + case "detect": + return encodeJSON(appverify.Detect(root)) + case "manifest": + r, existed, err := appverify.LoadOrDetect(root) + if err != nil { + return "", err + } + path := appverify.ManifestPath(root) + source := "detected now" + if existed { + source = "loaded existing" + } + out, err := encodeJSON(map[string]interface{}{"source": source, "path": path, "recipe": r}) + if err != nil { + return "", err + } + return out, nil + default: // smoke + return runBootSmoke(ctx, root, params.ReadinessSeconds) + } +} + +// runBootSmoke starts the recipe's start command, polls readiness until it +// succeeds or the budget expires, then always stops the process. Only fixed +// argv lists from the (normalized) recipe are executed — no shell. +func runBootSmoke(ctx context.Context, root string, readinessSeconds int) (string, error) { + r, _, err := appverify.LoadOrDetect(root) + if err != nil { + return "", err + } + res := smokeResult{SmokeKind: string(r.SmokeKind), Target: r.SmokeTarget()} + if len(r.Start) == 0 { + res.Status = "skipped" + res.Error = "no start command in recipe; run action=manifest after confirming how the app boots" + return encodeJSON(res) + } + res.StartCmd = joinArgs(r.Start) + if readinessSeconds <= 0 { + readinessSeconds = 60 + } + if readinessSeconds > 300 { + readinessSeconds = 300 + } + + runCtx, cancel := context.WithTimeout(ctx, time.Duration(readinessSeconds+30)*time.Second) + defer cancel() + started := time.Now() + + cmd := exec.CommandContext(runCtx, r.Start[0], r.Start[1:]...) // #nosec G204 -- fixed argv from normalized recipe; no shell + cmd.Dir = root + if err := cmd.Start(); err != nil { + res.Status = "failed" + res.Error = fmt.Sprintf("start failed: %v", err) + return encodeJSON(res) + } + // Always tear the process down before returning. + defer func() { + if cmd.Process != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + } + }() + + ready := false + switch r.SmokeKind { + case appverify.SmokeHTTP: + client := &http.Client{Timeout: 2 * time.Second} + deadline := time.Now().Add(time.Duration(readinessSeconds) * time.Second) + for time.Now().Before(deadline) { + req, err := http.NewRequestWithContext(runCtx, http.MethodGet, res.Target, nil) + if err != nil { + res.Error = fmt.Sprintf("build readiness request: %v", err) + break + } + resp, err := client.Do(req) + if err == nil { + _ = resp.Body.Close() + ready = true + break + } + select { + case <-runCtx.Done(): + case <-time.After(500 * time.Millisecond): + } + } + default: + // CLI/none: reaching here without the process exiting early is the best + // available signal for long-lived commands; short-lived ones are judged + // by Wait below. + ready = true + } + + exitErr := cmd.Wait() + res.Duration = time.Since(started).Round(time.Millisecond).String() + switch { + case r.SmokeKind == appverify.SmokeHTTP && ready && exitErr == nil: + res.Status = "passed" + case r.SmokeKind == appverify.SmokeCLI && exitErr == nil: + res.Status = "passed" + case exitErr != nil: + res.Status = "failed" + ee := &exec.ExitError{} + if errors.As(exitErr, &ee) { + res.Error = fmt.Sprintf("app exited with code %d before/during readiness", ee.ExitCode()) + } else { + res.Error = exitErr.Error() + } + default: + res.Status = "failed" + res.Error = "readiness not observed within budget" + } + return encodeJSON(res) +} + +func joinArgs(args []string) string { + out := "" + for i, a := range args { + if i > 0 { + out += " " + } + out += a + } + return out +} diff --git a/internal/tool/app_verify_test.go b/internal/tool/app_verify_test.go new file mode 100644 index 00000000..9c79c05b --- /dev/null +++ b/internal/tool/app_verify_test.go @@ -0,0 +1,96 @@ +package tool + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAppVerifyDetectAction(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module x\n\ngo 1.22\n"), 0o644); err != nil { + t.Fatal(err) + } + out, err := AppVerifyTool{}.Execute(context.Background(), json.RawMessage(`{"action":"detect","path":"`+dir+`"}`)) + if err != nil { + t.Fatalf("Execute: %v", err) + } + var recipe map[string]interface{} + if err := json.Unmarshal([]byte(out), &recipe); err != nil { + t.Fatalf("decode: %v", err) + } + if recipe["ecosystem"] != "go" { + t.Fatalf("ecosystem = %v", recipe["ecosystem"]) + } +} + +func TestAppVerifyManifestActionPersistsContract(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module x\n\ngo 1.22\n"), 0o644); err != nil { + t.Fatal(err) + } + out, err := AppVerifyTool{}.Execute(context.Background(), json.RawMessage(`{"action":"manifest","path":"`+dir+`"}`)) + if err != nil { + t.Fatalf("Execute: %v", err) + } + var resp struct { + Source string `json:"source"` + Path string `json:"path"` + } + if err := json.Unmarshal([]byte(out), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.Source != "detected now" || !strings.Contains(resp.Path, "environment.json") { + t.Fatalf("resp = %+v", resp) + } + if _, err := os.Stat(resp.Path); err != nil { + t.Fatalf("manifest missing: %v", err) + } + + // Second run loads the existing manifest. + out2, err := AppVerifyTool{}.Execute(context.Background(), json.RawMessage(`{"action":"manifest","path":"`+dir+`"}`)) + if err != nil { + t.Fatal(err) + } + var resp2 struct { + Source string + } + if err := json.Unmarshal([]byte(out2), &resp2); err != nil { + t.Fatal(err) + } + if resp2.Source != "loaded existing" { + t.Fatalf("source = %q, want loaded existing", resp2.Source) + } +} + +func TestAppVerifySmokeSkipsWithoutStartCommand(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module x\n\ngo 1.22\n"), 0o644); err != nil { + t.Fatal(err) + } + out, err := AppVerifyTool{}.Execute(context.Background(), json.RawMessage( + `{"action":"smoke","path":"`+dir+`","readiness_seconds":2}`, + )) + if err != nil { + t.Fatalf("Execute: %v", err) + } + var res smokeResult + if err := json.Unmarshal([]byte(out), &res); err != nil { + t.Fatalf("decode: %v", err) + } + // A go library has no start command: the tool must skip cleanly rather + // than fail or hang. + if res.Status != "skipped" { + t.Fatalf("status = %q (%s)", res.Status, res.Error) + } +} + +func TestAppVerifyInvalidAction(t *testing.T) { + tool := AppVerifyTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"action":"nope"}`)); err == nil { + t.Fatal("expected error for unsupported action") + } +}