diff --git a/cmd/chat_tools.go b/cmd/chat_tools.go index e792d939..6631b5bd 100644 --- a/cmd/chat_tools.go +++ b/cmd/chat_tools.go @@ -170,6 +170,7 @@ func optionalTools() []tool.Tool { tool.McpAuthTool{}, tool.DiagnosticsTool{}, tool.CodeSearchTool{}, + tool.CodeMatchTool{}, tool.CoreMemoryAppendTool{}, tool.CoreMemoryReplaceTool{}, tool.CoreMemoryRethinkTool{}, diff --git a/internal/engine/code_index_adapter.go b/internal/engine/code_index_adapter.go new file mode 100644 index 00000000..27859936 --- /dev/null +++ b/internal/engine/code_index_adapter.go @@ -0,0 +1,45 @@ +package engine + +import ( + "github.com/GrayCodeAI/hawk/internal/intelligence/memory" + "github.com/GrayCodeAI/hawk/internal/intelligence/repomap" +) + +// yaadCodeIndexer adapts *memory.YaadBridge to repomap.CodeIndexer. The two +// packages define identical CodeSearchResult shapes under different names, so +// the adapter only has to convert the SearchCode return type; every other +// method forwards directly. +type yaadCodeIndexer struct { + bridge *memory.YaadBridge +} + +func (a *yaadCodeIndexer) IndexCodeChunk(path, content, symbol, lang string, start, end, tokens int, hash string) error { + return a.bridge.IndexCodeChunk(path, content, symbol, lang, start, end, tokens, hash) +} + +func (a *yaadCodeIndexer) SearchCode(query string, limit int) ([]repomap.CodeSearchResult, error) { + results, err := a.bridge.SearchCode(query, limit) + if err != nil { + return nil, err + } + out := make([]repomap.CodeSearchResult, 0, len(results)) + for _, r := range results { + out = append(out, repomap.CodeSearchResult{ + Path: r.Path, StartLine: r.StartLine, EndLine: r.EndLine, + Content: r.Content, Symbol: r.Symbol, Score: r.Score, + }) + } + return out, nil +} + +func (a *yaadCodeIndexer) GetFileHash(path string) (string, error) { + return a.bridge.GetFileHash(path) +} + +func (a *yaadCodeIndexer) ClearFileChunks(path string) error { + return a.bridge.ClearFileChunks(path) +} + +func (a *yaadCodeIndexer) ListIndexedPaths() ([]string, error) { + return a.bridge.ListIndexedPaths() +} diff --git a/internal/engine/safety/capabilities.go b/internal/engine/safety/capabilities.go index 6fe34d31..4dd4b542 100644 --- a/internal/engine/safety/capabilities.go +++ b/internal/engine/safety/capabilities.go @@ -46,6 +46,7 @@ var toolPolicies = map[string]ToolPolicy{ "Outline": {Name: "Outline", Capabilities: []Capability{CapabilityFilesystemRead}, DefaultRisk: RiskLow}, "SmartRead": {Name: "SmartRead", Capabilities: []Capability{CapabilityFilesystemRead}, DefaultRisk: RiskLow}, "CodeSearch": {Name: "CodeSearch", Capabilities: []Capability{CapabilityFilesystemRead}, DefaultRisk: RiskLow}, + "CodeMatch": {Name: "CodeMatch", Capabilities: []Capability{CapabilityFilesystemRead}, DefaultRisk: RiskLow}, "CodeGraph": {Name: "CodeGraph", Capabilities: []Capability{CapabilityFilesystemRead}, DefaultRisk: RiskLow}, "Impact": {Name: "Impact", Capabilities: []Capability{CapabilityFilesystemRead}, DefaultRisk: RiskLow}, "GitHistory": {Name: "GitHistory", Capabilities: []Capability{CapabilityFilesystemRead, CapabilityProcessExecute}, DefaultRisk: RiskLow}, diff --git a/internal/engine/safety/permission.go b/internal/engine/safety/permission.go index eefef55d..2a772414 100644 --- a/internal/engine/safety/permission.go +++ b/internal/engine/safety/permission.go @@ -315,6 +315,8 @@ func canonicalToolName(name string) string { return "WebFetch" case "web_search", "websearch": return "WebSearch" + case "code_match", "codematch", "match_code": + return "CodeMatch" case "tool_health", "toolhealth", "tools_health": return "ToolHealth" case "project_verify", "projectverify", "verify_project": diff --git a/internal/engine/tool_service.go b/internal/engine/tool_service.go index b32b9ba5..f390b491 100644 --- a/internal/engine/tool_service.go +++ b/internal/engine/tool_service.go @@ -12,6 +12,7 @@ import ( "github.com/GrayCodeAI/hawk/internal/engine/diff" "github.com/GrayCodeAI/hawk/internal/hooks" "github.com/GrayCodeAI/hawk/internal/intelligence/memory" + "github.com/GrayCodeAI/hawk/internal/intelligence/repomap" "github.com/GrayCodeAI/hawk/internal/observability/metrics" "github.com/GrayCodeAI/hawk/internal/observability/oteltrace" "github.com/GrayCodeAI/hawk/internal/prompts" @@ -478,17 +479,52 @@ func (s *ToolService) ExecuteOne(ctx context.Context, tc types.ToolCall, overrid AskUserFn: s.deps.askUser, CommitMessageChatFn: commitChat, YaadBridge: yaad, - SpecSlugGet: func() string { return s.deps.permissions.SpecSlug() }, - SpecSlugSet: func(slug string) { s.deps.permissions.SetSpecSlug(slug) }, - AllowedDirectories: s.deps.permissions.AllowedDirs(), - SandboxMode: sbMode, - BackgroundManager: s.EnsureBackgroundManager(), - ReadOnlyBash: s.ReadOnlyBash(), - WorkingDir: s.WorkingDir(), - AvailableTools: available, - Registry: s.registry, - AutoCommit: s.AutoCommit(), - TaskExecutor: s.deps.taskExec, + // Semantic code search backed by the yaad code-chunk index. Wiring the + // closures here makes CodeSearchTool functional in production (the + // interface was declared but never bound). Refresh rebuilds only + // added/changed files via content-hash staleness. + CodeSearchFn: func(cctx context.Context, query string, limit int) ([]tool.CodeSearchResult, error) { + if yaad == nil { + return nil, fmt.Errorf("code search unavailable: no memory bridge") + } + results, err := yaad.SearchCode(query, limit) + if err != nil { + return nil, err + } + out := make([]tool.CodeSearchResult, 0, len(results)) + for _, r := range results { + out = append(out, tool.CodeSearchResult{ + Path: r.Path, StartLine: r.StartLine, EndLine: r.EndLine, + Content: r.Content, Symbol: r.Symbol, Language: tool.LanguageForFile(r.Path), Score: r.Score, + }) + } + return out, nil + }, + RefreshCodeIndexFn: func(cctx context.Context) error { + if yaad == nil { + return fmt.Errorf("code index refresh unavailable: no memory bridge") + } + dir := s.WorkingDir() + if dir == "" { + return fmt.Errorf("code index refresh unavailable: no working directory") + } + if err := yaad.InitCodeIndex(); err != nil { + return err + } + _, _, _, err := repomap.IncrementalReindex(dir, nil, &yaadCodeIndexer{yaad}) + return err + }, + SpecSlugGet: func() string { return s.deps.permissions.SpecSlug() }, + SpecSlugSet: func(slug string) { s.deps.permissions.SetSpecSlug(slug) }, + AllowedDirectories: s.deps.permissions.AllowedDirs(), + SandboxMode: sbMode, + BackgroundManager: s.EnsureBackgroundManager(), + ReadOnlyBash: s.ReadOnlyBash(), + WorkingDir: s.WorkingDir(), + AvailableTools: available, + Registry: s.registry, + AutoCommit: s.AutoCommit(), + TaskExecutor: s.deps.taskExec, }) // Bridge session sandbox policy onto the context so Bash/PowerShell // WrapCommand actually applies. Path guards already read ToolContext.SandboxMode; diff --git a/internal/tool/code_match.go b/internal/tool/code_match.go new file mode 100644 index 00000000..3a119dcd --- /dev/null +++ b/internal/tool/code_match.go @@ -0,0 +1,118 @@ +package tool + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// CodeMatchTool performs structural, AST-level code search using tree-sitter +// query patterns — the hawk counterpart to cocoindex-code's by-example +// structural grep. Patterns are tree-sitter query S-expressions over the +// language grammar, e.g. for Go: +// +// (function_declaration name: (identifier) @name) @fn +// +// Captures (@name) surface per match so callers extract identifiers without +// regex fragility. Unlike Grep (textual), matches are syntax-aware: comments +// and strings cannot produce false positives. +type CodeMatchTool struct{} + +func (CodeMatchTool) Name() string { return "CodeMatch" } +func (CodeMatchTool) RiskLevel() string { return "low" } +func (CodeMatchTool) Aliases() []string { return []string{"code_match", "match_code"} } + +func (CodeMatchTool) Description() string { + return `Structural code search with tree-sitter query patterns. Matches the AST, not text: comments/strings cannot false-positive. Pattern is a tree-sitter query S-expression; use @captures to extract parts. Examples - Go functions: "(function_declaration name: (identifier) @name) @fn" | Go calls of one function: "(call_expression function: (identifier) @callee) @call" | Python defs: "(function_definition name: (identifier) @name) @fn". Language auto-detects per file; restrict with language=go|python|typescript|tsx.` +} + +func (CodeMatchTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "pattern": map[string]interface{}{ + "type": "string", + "description": "Tree-sitter query pattern (S-expression). Captures (@name) are returned per match.", + }, + "path": map[string]interface{}{ + "type": "string", + "description": "Project directory (default: session working directory).", + }, + "language": map[string]interface{}{ + "type": "string", + "enum": []string{"go", "python", "typescript", "tsx"}, + "description": "Restrict to one language (default: all supported).", + }, + "limit": map[string]interface{}{ + "type": "integer", + "minimum": 1, + "maximum": 200, + "description": "Maximum matches total (default 30).", + }, + }, + "required": []string{"pattern"}, + } +} + +// codeMatchHit is one structural match. +type codeMatchHit struct { + File string `json:"file"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + Captures []string `json:"captures,omitempty"` + Snippet string `json:"snippet"` +} + +// codeMatchLanguages maps file extensions to supported grammar names. +var codeMatchExtLang = map[string]string{ + ".go": "go", ".py": "python", + ".ts": "typescript", ".tsx": "tsx", ".mts": "typescript", ".cts": "typescript", +} + +func (CodeMatchTool) Execute(ctx context.Context, input json.RawMessage) (string, error) { + var params struct { + Pattern string `json:"pattern"` + Path string `json:"path"` + Language string `json:"language"` + Limit int `json:"limit"` + } + if err := json.Unmarshal(input, ¶ms); err != nil { + return "", fmt.Errorf("invalid input: %w", err) + } + params.Pattern = strings.TrimSpace(params.Pattern) + if params.Pattern == "" { + return "", fmt.Errorf("pattern is required") + } + params.Language = strings.ToLower(strings.TrimSpace(params.Language)) + + root := params.Path + if root == "" { + if tc := GetToolContext(ctx); tc != nil && tc.WorkingDir != "" { + root = tc.WorkingDir + } else { + var err error + root, err = os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve working directory: %w", err) + } + } + } + absRoot, err := filepath.Abs(root) + if err != nil { + return "", fmt.Errorf("resolve project path: %w", err) + } + if err := validatePathAllowed(ctx, absRoot); err != nil { + return "", err + } + if params.Limit <= 0 { + params.Limit = 30 + } + if params.Limit > 200 { + params.Limit = 200 + } + + return runCodeMatch(ctx, absRoot, params.Pattern, params.Language, params.Limit) +} diff --git a/internal/tool/code_match_cgo.go b/internal/tool/code_match_cgo.go new file mode 100644 index 00000000..944ce1ce --- /dev/null +++ b/internal/tool/code_match_cgo.go @@ -0,0 +1,239 @@ +//go:build cgo + +package tool + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + sitter "github.com/smacker/go-tree-sitter" + "github.com/smacker/go-tree-sitter/golang" + "github.com/smacker/go-tree-sitter/python" + "github.com/smacker/go-tree-sitter/typescript/tsx" + "github.com/smacker/go-tree-sitter/typescript/typescript" +) + +// matchEngine implements structural matching over the same tree-sitter +// grammars codegraph uses. One parser per language, reused across files. +type matchEngine struct { + mu sync.Mutex + parsers map[string]*sitter.Parser + languages map[string]*sitter.Language +} + +func newMatchEngine() *matchEngine { + return &matchEngine{ + parsers: map[string]*sitter.Parser{}, + languages: map[string]*sitter.Language{}, + } +} + +func (e *matchEngine) lang(name string) (*sitter.Language, bool) { + e.mu.Lock() + defer e.mu.Unlock() + if l, ok := e.languages[name]; ok { + return l, true + } + var l *sitter.Language + switch name { + case "go": + l = golang.GetLanguage() + case "python": + l = python.GetLanguage() + case "typescript": + l = typescript.GetLanguage() + case "tsx": + l = tsx.GetLanguage() + default: + return nil, false + } + e.languages[name] = l + return l, true +} + +func (e *matchEngine) parser(name string) (*sitter.Parser, bool) { + e.mu.Lock() + if p, ok := e.parsers[name]; ok { + e.mu.Unlock() + return p, true + } + e.mu.Unlock() + + l, ok := e.lang(name) + if !ok { + return nil, false + } + p := sitter.NewParser() + p.SetLanguage(l) + e.mu.Lock() + if existing, exists := e.parsers[name]; exists { + e.mu.Unlock() + p.Close() + return existing, true + } + e.parsers[name] = p + e.mu.Unlock() + return p, true +} + +// compileQuery compiles a tree-sitter query pattern for one language. +func (e *matchEngine) compileQuery(langName, pattern string) (*sitter.Query, error) { + l, ok := e.lang(langName) + if !ok { + return nil, fmt.Errorf("unsupported language %q", langName) + } + q, err := sitter.NewQuery([]byte(pattern), l) + if err != nil { + return nil, fmt.Errorf("invalid pattern for %s: %w", langName, err) + } + return q, nil +} + +// matchSource runs the query over one source buffer and returns hits. +func (e *matchEngine) matchSource(ctx context.Context, langName string, q *sitter.Query, path, src string, limit int) ([]codeMatchHit, error) { + parser, ok := e.parser(langName) + if !ok { + return nil, fmt.Errorf("unsupported language %q", langName) + } + tree, err := parser.ParseCtx(ctx, nil, []byte(src)) + if err != nil { + return nil, err + } + defer tree.Close() + + qc := sitter.NewQueryCursor() + defer qc.Close() + qc.Exec(q, tree.RootNode()) + + lines := strings.Split(src, "\n") + var hits []codeMatchHit + for { + m, ok := qc.NextMatch() + if !ok || (limit > 0 && len(hits) >= limit) { + break + } + if len(m.Captures) == 0 { + continue + } + startByte := int(m.Captures[0].Node.StartByte()) + endByte := int(m.Captures[0].Node.EndByte()) + startLine := int(m.Captures[0].Node.StartPoint().Row) + 1 + endLine := int(m.Captures[0].Node.EndPoint().Row) + 1 + + names := make([]string, 0, len(m.Captures)) + seen := map[string]bool{} + for _, c := range m.Captures { + if c.Node == nil { + continue + } + cname := q.CaptureNameForId(c.Index) + if seen[cname] { + continue + } + seen[cname] = true + names = append(names, cname) + if int(c.Node.StartByte()) < startByte { + startByte = int(c.Node.StartByte()) + startLine = int(c.Node.StartPoint().Row) + 1 + } + if int(c.Node.EndByte()) > endByte { + endByte = int(c.Node.EndByte()) + endLine = int(c.Node.EndPoint().Row) + 1 + } + } + + const maxSnippetLines = 12 + lo := startLine - 1 + if lo < 0 { + lo = 0 + } + if endLine > len(lines) { + endLine = len(lines) + } + snippetLines := lines[lo:endLine] + if len(snippetLines) > maxSnippetLines { + snippetLines = append(append([]string{}, snippetLines[:maxSnippetLines]...), "... (truncated)") + } + + hits = append(hits, codeMatchHit{ + File: path, + StartLine: startLine, + EndLine: endLine, + Captures: names, + Snippet: strings.Join(snippetLines, "\n"), + }) + } + return hits, nil +} + +func runCodeMatch(ctx context.Context, root, pattern, language string, limit int) (string, error) { + engine := newMatchEngine() + langs := []string{} + if language != "" { + if _, ok := engine.lang(language); !ok { + return "", fmt.Errorf("unsupported language %q (supported: go, python, typescript, tsx)", language) + } + langs = append(langs, language) + } else { + langs = append(langs, "go", "python", "typescript", "tsx") + } + queries := make(map[string]*sitter.Query, len(langs)) + for _, ln := range langs { + q, err := engine.compileQuery(ln, pattern) + if err != nil { + return "", err + } + queries[ln] = q + defer q.Close() + } + var allHits []codeMatchHit + filesScanned := 0 + walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, werr error) error { + if werr != nil { + return nil + } + if d.IsDir() { + name := d.Name() + if name == ".git" || name == "node_modules" || name == "vendor" || name == ".hawk" || name == "__pycache__" || name == "dist" || name == "target" { + return filepath.SkipDir + } + return nil + } + langName, ok := codeMatchExtLang[strings.ToLower(filepath.Ext(path))] + if !ok || queries[langName] == nil || len(allHits) >= limit { + return nil + } + src, err := os.ReadFile(path) // #nosec G304 -- workspace-relative walk path + if err != nil { + return nil + } + filesScanned++ + hits, err := engine.matchSource(ctx, langName, queries[langName], path, string(src), limit-len(allHits)) + if err == nil { + allHits = append(allHits, hits...) + } + return nil + }) + if walkErr != nil { + return "", fmt.Errorf("walk: %w", walkErr) + } + sort.SliceStable(allHits, func(i, j int) bool { + if allHits[i].File != allHits[j].File { + return allHits[i].File < allHits[j].File + } + return allHits[i].StartLine < allHits[j].StartLine + }) + payload := map[string]interface{}{"pattern": pattern, "files_scanned": filesScanned, "matches": len(allHits), "truncated": len(allHits) >= limit, "hits": allHits} + out, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return "", err + } + return string(out), nil +} diff --git a/internal/tool/code_match_nocgo.go b/internal/tool/code_match_nocgo.go new file mode 100644 index 00000000..544ca2ac --- /dev/null +++ b/internal/tool/code_match_nocgo.go @@ -0,0 +1,18 @@ +//go:build !cgo + +package tool + +import ( + "context" + "fmt" +) + +// matchSourceWithEngine is unavailable without cgo; the tool reports the +// limitation instead of pretending to search. +func runCodeMatch(ctx context.Context, root, pattern, language string, limit int) (string, error) { + return "", fmt.Errorf("CodeMatch requires a cgo build (tree-sitter grammars are linked at compile time)") +} + +type matchEngine struct{} + +func newMatchEngine() *matchEngine { return &matchEngine{} } diff --git a/internal/tool/code_match_test.go b/internal/tool/code_match_test.go new file mode 100644 index 00000000..a625b453 --- /dev/null +++ b/internal/tool/code_match_test.go @@ -0,0 +1,129 @@ +package tool + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeMatchTree(t *testing.T, files map[string]string) string { + t.Helper() + root := t.TempDir() + for rel, content := range files { + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +func TestCodeMatchGoFunctionPattern(t *testing.T) { + root := writeMatchTree(t, map[string]string{ + "a/a.go": "package a\n\n// helper comment mentioning Handler\nfunc Handler(w io.Writer) { }\n\nfunc ignored() {}\n", + }) + out, err := CodeMatchTool{}.Execute(context.Background(), json.RawMessage( + `{"pattern":"(function_declaration name: (identifier) @name) @fn","path":"`+root+`","language":"go"}`, + )) + if err != nil { + t.Fatalf("Execute: %v", err) + } + var resp struct { + Matches int `json:"matches"` + Hits []struct { + File string `json:"file"` + StartLine int `json:"start_line"` + Captures []string `json:"captures"` + Snippet string `json:"snippet"` + } `json:"hits"` + } + if err := json.Unmarshal([]byte(out), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.Matches != 2 { + t.Fatalf("matches = %d (%s)", resp.Matches, out) + } + if len(resp.Hits[0].Captures) == 0 { + t.Fatal("expected captures in hits") + } +} + +func TestCodeMatchCommentsDoNotFalsePositive(t *testing.T) { + root := writeMatchTree(t, map[string]string{ + "a/a.go": "package a\n\n// func fakeDeclaration name: (identifier)\nfunc Real() {}\n", + }) + out, err := CodeMatchTool{}.Execute(context.Background(), json.RawMessage( + `{"pattern":"(function_declaration name: (identifier) @n) @f","path":"`+root+`","language":"go","limit":10}`, + )) + if err != nil { + t.Fatal(err) + } + if strings.Count(out, `"start_line"`) != 1 { + t.Fatalf("comment produced a structural match: %s", out) + } +} + +func TestCodeMatchPythonDef(t *testing.T) { + root := writeMatchTree(t, map[string]string{ + "svc.py": "def handler(req):\n return req\n\nclass C:\n def method(self):\n pass\n", + }) + out, err := CodeMatchTool{}.Execute(context.Background(), json.RawMessage( + `{"pattern":"(function_definition name: (identifier) @name) @fn","path":"`+root+`","language":"python"}`, + )) + if err != nil { + t.Fatal(err) + } + var resp struct { + Matches int `json:"matches"` + } + _ = json.Unmarshal([]byte(out), &resp) + if resp.Matches != 2 { // handler + method + t.Fatalf("python defs matched = %d (%s)", resp.Matches, out) + } +} + +func TestCodeMatchLanguageFilterAndLimit(t *testing.T) { + root := writeMatchTree(t, map[string]string{ + "x.go": "package x\nfunc A() {}\nfunc B() {}\nfunc C() {}\n", + }) + out, err := CodeMatchTool{}.Execute(context.Background(), json.RawMessage( + `{"pattern":"(function_declaration) @f","path":"`+root+`","language":"go","limit":2}`, + )) + if err != nil { + t.Fatal(err) + } + var resp struct { + Matches int `json:"matches"` + Truncated bool `json:"truncated"` + } + _ = json.Unmarshal([]byte(out), &resp) + if resp.Matches != 2 || !resp.Truncated { + t.Fatalf("limit not applied: matches=%d truncated=%v", resp.Matches, resp.Truncated) + } +} + +func TestCodeMatchInvalidPatternFailsBeforeWalk(t *testing.T) { + root := t.TempDir() + tool := CodeMatchTool{} + _, err := tool.Execute(context.Background(), json.RawMessage( + `{"pattern":"((( not-a-query","path":"`+root+`","language":"go"}`, + )) + if err == nil { + t.Fatal("invalid pattern must error before scanning") + } +} + +func TestCodeMatchUnsupportedLanguage(t *testing.T) { + tool := CodeMatchTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage( + `{"pattern":"(x)","language":"ruby"}`, + )); err == nil { + t.Fatal("unsupported language must error") + } +}