feat(tools): opt-in tool-catalog shrink via tok; bump submodule to #83

internal/engine/chat_service.go

@@ -207,6 +207,9 @@ func (c *ChatService) BuildOptions(systemPrompt, activeModel string, maxTokens i
 	if outputSchema != "" {
 		opts.ResponseFormat = &types.ResponseFormat{Type: "json_schema", Schema: outputSchema}
 	}
+	// Opt-in tool-catalog compression (HAWK_TOOL_SHRINK=1): fail-open, so the
+	// returned tools equal the input whenever anything is off or drifts.
+	opts.Tools = shrinkEyrieTools(opts.Tools)
 	return opts
 }
 

internal/engine/token/tok_facade.go

@@ -49,5 +49,15 @@ func Compress(text string, budget int) (string, Stats) {
 // JSONInvariants renders verified-fact summaries for elided JSON records.
 func JSONInvariants(dropped []json.RawMessage) string { return hawktoken.JSONInvariants(dropped) }
 
+// ShrinkToolCatalog compresses an OpenAI-style function-tool catalog,
+// preserving the selection surface byte-for-byte. Fail-open: unchanged input
+// with ok=false when nothing can be safely reduced.
+func ShrinkToolCatalog(catalog string) (string, bool) { return hawktoken.ShrinkToolCatalog(catalog) }
+
+// LintToolCatalog reports per-tool reductions without committing.
+func LintToolCatalog(catalog string) ([]hawktoken.ToolShrinkStats, bool) {
+	return hawktoken.LintToolCatalog(catalog)
+}
+
 // LogInvariants renders the level distribution of elided log lines.
 func LogInvariants(lines []string) string { return hawktoken.LogInvariants(lines) }

internal/engine/tool_catalog_shrink.go

@@ -0,0 +1,103 @@
+package engine
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/GrayCodeAI/hawk/internal/engine/token"
+	"github.com/GrayCodeAI/hawk/internal/storage"
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+// Tool-catalog shrink, adopted from caveman's toolschema compressor (via
+// tok): large tool catalogs are paid on every request, so when enabled the
+// outgoing catalog is compressed with a strict over-keep contract — names,
+// types, enums, required, defaults survive byte-for-byte; only annotation
+// metadata is dropped and long descriptions reduced to lead+constraint
+// sentences. Fail-open throughout: anything that does not round-trip
+// cleanly falls back to the original catalog.
+
+// toolShrinkEnabled reports whether opt-in catalog compression is on
+// (HAWK_TOOL_SHRINK=1). Default off: existing request bytes unchanged.
+func toolShrinkEnabled() bool {
+	return strings.EqualFold(os.Getenv("HAWK_TOOL_SHRINK"), "1")
+}
+
+// originalsDir stores pre-shrink catalogs keyed by content hash so the exact
+// original surface stays recoverable for debugging and diffing.
+func originalsDir() string {
+	return filepath.Join(storage.StateDir(), "tool-catalog-originals")
+}
+
+// shrinkEyrieTools compresses the hawk tool list via tok's toolschema
+// compressor. The list is converted to the OpenAI function-catalog wire shape,
+// shrunk, and converted back; any name-set mismatch fails open to input.
+// When compression changed something, the original catalog is persisted under
+// the state dir keyed by content hash before the shrunk form is returned.
+func shrinkEyrieTools(tools []types.EyrieTool) []types.EyrieTool {
+	if len(tools) == 0 || !toolShrinkEnabled() {
+		return tools
+	}
+
+	type wireTool struct {
+		Type     string          `json:"type"`
+		Function types.EyrieTool `json:"function"`
+	}
+	wire := make([]wireTool, len(tools))
+	for i := range tools {
+		wire[i] = wireTool{Type: "function", Function: tools[i]}
+	}
+	raw, err := json.Marshal(wire)
+	if err != nil {
+		return tools
+	}
+
+	shrunk, changed := token.ShrinkToolCatalog(string(raw))
+	if !changed {
+		return tools
+	}
+	var shrunkWire []wireTool
+	if err := json.Unmarshal([]byte(shrunk), &shrunkWire); err != nil {
+		return tools
+	}
+	if len(shrunkWire) != len(tools) {
+		return tools // structural drift: never risk it
+	}
+	out := make([]types.EyrieTool, len(tools))
+	for i := range shrunkWire {
+		if shrunkWire[i].Function.Name != tools[i].Name {
+			slog.Debug("tool shrink name drift, failing open", "position", i)
+			return tools
+		}
+		out[i] = shrunkWire[i].Function
+	}
+
+	persistOriginalCatalog(raw)
+	slog.Info(
+		"tool catalog shrunk",
+		"tools", len(tools),
+		"bytes_before", len(raw),
+		"bytes_after", len(shrunk),
+	)
+	return out
+}
+
+// persistOriginalCatalog writes the pre-shrink catalog once per content hash.
+func persistOriginalCatalog(raw []byte) {
+	sum := sha256.Sum256(raw)
+	name := hex.EncodeToString(sum[:8]) + ".json"
+	dir := originalsDir()
+	path := filepath.Join(dir, name)
+	if _, err := os.Stat(path); err == nil {
+		return
+	}
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return
+	}
+	_ = os.WriteFile(path, raw, 0o600) // #nosec G306 -- session-local recovery copy
+}

internal/engine/tool_catalog_shrink_test.go

@@ -0,0 +1,90 @@
+package engine
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+func bloatedTools() []types.EyrieTool {
+	return []types.EyrieTool{
+		{
+			Name: "read_file",
+			Description: strings.Repeat("Reads a file from disk quickly and safely. ", 30) +
+				"The path must be an absolute path. You cannot read binary files.",
+			Parameters: map[string]interface{}{
+				"type":    "object",
+				"$schema": "http://json-schema.org/draft-07/schema#",
+				"title":   "params",
+				"properties": map[string]interface{}{
+					"path": map[string]interface{}{"type": "string"},
+				},
+				"required": []string{"path"},
+			},
+		},
+		{
+			Name:        "tiny_tool",
+			Description: "ok",
+			Parameters:  map[string]interface{}{"type": "object"},
+		},
+	}
+}
+
+func TestShrinkEyrieToolsDisabledByDefault(t *testing.T) {
+	t.Setenv("HAWK_TOOL_SHRINK", "")
+	in := bloatedTools()
+	out := shrinkEyrieTools(in)
+	if len(out) != len(in) || out[0].Description != in[0].Description {
+		t.Fatal("shrink must be a no-op when disabled")
+	}
+}
+
+func TestShrinkEyrieToolsEnabledReducesAndPreservesNames(t *testing.T) {
+	t.Setenv("HAWK_TOOL_SHRINK", "1")
+	t.Setenv("HAWK_STATE_DIR", t.TempDir())
+	in := bloatedTools()
+	out := shrinkEyrieTools(in)
+	if len(out) != 2 {
+		t.Fatalf("tool count changed: %d", len(out))
+	}
+	if out[0].Name != "read_file" || out[1].Name != "tiny_tool" {
+		t.Fatalf("names drifted: %q %q", out[0].Name, out[1].Name)
+	}
+	if len(out[0].Description) >= len(in[0].Description) {
+		t.Fatalf("description not reduced: %d vs %d", len(out[0].Description), len(in[0].Description))
+	}
+	if !strings.Contains(out[0].Description, "must be an absolute path") {
+		t.Fatalf("constraint sentence lost: %q", out[0].Description)
+	}
+	// required survived through the schema tree
+	if req, ok := out[0].Parameters["required"]; !ok || req == nil {
+		t.Fatalf("required dropped: %+v", out[0].Parameters)
+	}
+}
+
+func TestBuildOptionsAppliesShrink(t *testing.T) {
+	t.Setenv("HAWK_TOOL_SHRINK", "1")
+	t.Setenv("HAWK_STATE_DIR", t.TempDir())
+	c := &ChatService{}
+	opts := c.BuildOptions("sys", "m", 100, bloatedTools())
+	if len(opts.Tools) != 2 || opts.Tools[0].Name != "read_file" {
+		t.Fatalf("tools = %+v", opts.Tools)
+	}
+	if len(opts.Tools[0].Description) >= len(bloatedTools()[0].Description) {
+		t.Fatal("BuildOptions did not shrink the catalog")
+	}
+}
+
+func TestOriginalCatalogPersistedForRecovery(t *testing.T) {
+	stateDir := t.TempDir()
+	t.Setenv("HAWK_TOOL_SHRINK", "1")
+	t.Setenv("HAWK_STATE_DIR", stateDir)
+	in := bloatedTools()
+	_ = shrinkEyrieTools(in)
+	matches, err := filepath.Glob(stateDir + "/tool-catalog-originals/*.json")
+	if err != nil || len(matches) == 0 {
+		t.Fatalf("original catalog not persisted: %v %v", matches, err)
+	}
+}

internal/token/tok.go

@@ -36,8 +36,21 @@ func NewUsageTracker() *UsageTracker { return tok.NewUsageTracker() }
 // JSONInvariants renders verified-fact summaries for elided JSON records
 // (constants, enumerations, ranges, coverage). "" when nothing clears the
 // withhold rules.
+// ToolShrinkStats reports one tool's catalog reduction.
+type ToolShrinkStats = tok.ToolShrinkStats
+
 func JSONInvariants(dropped []json.RawMessage) string { return tok.JSONInvariants(dropped) }
 
+// ShrinkToolCatalog compresses an OpenAI-style function-tool catalog,
+// preserving the selection surface byte-for-byte. Fail-open: unchanged input
+// with ok=false when nothing can be safely reduced.
+func ShrinkToolCatalog(catalog string) (string, bool) { return tok.ShrinkToolCatalog(catalog) }
+
+// LintToolCatalog reports per-tool reductions without committing.
+func LintToolCatalog(catalog string) ([]tok.ToolShrinkStats, bool) {
+	return tok.LintToolCatalog(catalog)
+}
+
 // LogInvariants renders the level distribution of elided log lines.
 // "" when the lines do not parse as logs.
 func LogInvariants(lines []string) string { return tok.LogInvariants(lines) }