From 3e5a0645bbd9bcb16d3c2a984e46b3b01618da62 Mon Sep 17 00:00:00 2001 From: Lisa Date: Wed, 15 Apr 2026 20:41:43 +0200 Subject: [PATCH] feat(lip): wire up stream_context, query_expansion, explain_match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the correctness and utilisation gaps found in the CKB 9.1.0 review. Correctness: - lipFileURI now handles absolute paths and already-prefixed file:// URIs instead of naively joining them with repoRoot. - Handshake runs once on engine startup; its supported_messages list drives Engine.lipSupports(), the gate for v2.0+ RPCs against older daemons. Daemon version + supported_messages length are logged. Utilisation (three high-ROI LIP RPCs we were not using): - stream_context (v2.1) → explainFile now attaches a ranked list of semantically-related symbols (top 10 within a 2048-token budget) to the response's facts.related field. New streaming transport in internal/lip/stream_context.go reads N symbol_info frames + the end_stream terminator — the previous LIP client was unary-only. - query_expansion (v1.6) → SearchSymbols expands ≤ 2-token queries with up to 5 related terms before FTS5. Recovers recall on vocabulary-mismatch misses without touching precision on compound queries. - explain_match (v2.0) → SemanticSearchWithLIPExplained attaches up to two ranked chunks per semantic hit (top 5 hits, line ranges + text + score), letting the caller cite specific lines instead of a bare file URL. All three are gated on the handshake's supported_messages so clients talking to older daemons fall through to the legacy paths cleanly. Tests: unit coverage for StreamContext happy path, daemon-down, and error-frame abort. Existing lip_health, lip_ranker, and query tests still pass. Co-Authored-By: Claude Opus 4.6 --- CHANGELOG.md | 38 +++++++ internal/lip/client.go | 15 ++- internal/lip/stream_context.go | 148 +++++++++++++++++++++++++++ internal/lip/stream_context_test.go | 143 ++++++++++++++++++++++++++ internal/query/engine.go | 7 ++ internal/query/lip_health.go | 51 +++++++++ internal/query/lip_ranker.go | 72 ++++++++++++- internal/query/lip_stream_context.go | 59 +++++++++++ internal/query/navigation.go | 27 +++++ internal/query/query_expansion.go | 66 ++++++++++++ internal/query/symbols.go | 30 +++++- 11 files changed, 650 insertions(+), 6 deletions(-) create mode 100644 internal/lip/stream_context.go create mode 100644 internal/lip/stream_context_test.go create mode 100644 internal/query/lip_stream_context.go create mode 100644 internal/query/query_expansion.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 841fb0f0..a0fbe892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,44 @@ All notable changes to CKB will be documented in this file. ## [Unreleased] +### Added + +- **`explainFile` surfaces semantically-related symbols** via LIP v2.1's + `stream_context` RPC (`internal/query/lip_stream_context.go`). The daemon + ranks symbols across the whole file within a 2048-token budget; CKB + returns the top 10 in the new `facts.related` field with per-symbol + relevance and token cost. Gated on the handshake's `supported_messages` + — older daemons fall through and the field is absent. New streaming + transport `internal/lip/stream_context.go` reads the daemon's N + `symbol_info` frames plus the `end_stream` terminator; previous LIP + client was unary-only. +- **`searchSymbols` expands short queries** via LIP's `query_expansion` + RPC (`internal/query/query_expansion.go`). Queries of ≤ 2 tokens get up + to 5 related terms appended before hitting FTS5, recovering recall on + vocabulary-mismatch misses ("auth" → "authenticate authorization + principal…"). Gated on the handshake and on the same mixed-models flag + that protects the rerank path. Longer queries are passed through + unchanged — the expansion is a rescue, not a rewrite. +- **Semantic hits carry evidence chunks** when LIP v2.0+'s `explain_match` + is advertised (`SemanticSearchWithLIPExplained` in + `internal/query/lip_ranker.go`). Each hit returned by the semantic + fallback path now includes up to two ranked chunks with line ranges, + text, and per-chunk scores — the caller can cite specific lines instead + of a bare file URL. Capped at the top-5 hits to bound round-trip cost. +- **`lip.Handshake` runs on engine startup** and the daemon's + `supported_messages` list is stashed for feature gating + (`Engine.lipSupports`). The daemon version and supported-count are + logged on connect. + +### Changed + +- **`lipFileURI` path normalisation** — the helper that builds + `file://`-URIs for LIP requests used to naive-`filepath.Join` whatever + `Location.FileId` a backend supplied. Now handles absolute paths and + already-prefixed `file://` URIs without producing malformed results + like `file:///repo//abs/path`. Backends today return relative paths, so + this is a hardening fix for contracts that are nominally open. + ### Changed - **LIP health: push-driven, not polled** — the Engine now opens a long-lived diff --git a/internal/lip/client.go b/internal/lip/client.go index d8c51c1e..f23d094e 100644 --- a/internal/lip/client.go +++ b/internal/lip/client.go @@ -41,6 +41,10 @@ type NearestResult struct { type HandshakeInfo struct { DaemonVersion string `json:"daemon_version"` ProtocolVersion int `json:"protocol_version"` + // SupportedMessages is the snake_case `type` tag list the daemon + // understands. Empty when talking to a pre-v1.5 daemon that omits the + // field — callers should fall back to ProtocolVersion comparisons. + SupportedMessages []string `json:"supported_messages"` } // IndexStatusInfo is the public view of LIP index health. @@ -158,8 +162,9 @@ type batchAnnotationResp struct { } type handshakeResp struct { - DaemonVersion string `json:"daemon_version"` - ProtocolVersion int `json:"protocol_version"` + DaemonVersion string `json:"daemon_version"` + ProtocolVersion int `json:"protocol_version"` + SupportedMessages []string `json:"supported_messages"` } type similarityResp struct { @@ -790,7 +795,11 @@ func Handshake(clientVersion string) (*HandshakeInfo, error) { } return lipRPC(req, 200*time.Millisecond, 4<<10, func(r handshakeResp) *HandshakeInfo { - return &HandshakeInfo{DaemonVersion: r.DaemonVersion, ProtocolVersion: r.ProtocolVersion} + return &HandshakeInfo{ + DaemonVersion: r.DaemonVersion, + ProtocolVersion: r.ProtocolVersion, + SupportedMessages: r.SupportedMessages, + } }) } diff --git a/internal/lip/stream_context.go b/internal/lip/stream_context.go new file mode 100644 index 00000000..d4bb42a4 --- /dev/null +++ b/internal/lip/stream_context.go @@ -0,0 +1,148 @@ +package lip + +import ( + "encoding/json" + "io" + "net" + "time" +) + +// StreamContextSymbol is one frame of a StreamContext response. The +// embedded `OwnedSymbolInfo` is flattened into the fields we actually +// consume in CKB — the full Rust struct carries many fields we don't +// need (telemetry, relationships, taint) and serialising them through +// `map[string]any` would be wasteful. +type StreamContextSymbol struct { + URI string `json:"uri"` + DisplayName string `json:"display_name"` + Kind string `json:"kind"` + RelevanceScore float32 `json:"relevance_score"` + TokenCost uint32 `json:"token_cost"` +} + +// StreamContextResult summarises a completed StreamContext stream. +// `Reason` is one of "token_budget" | "exhausted" | "error". +type StreamContextResult struct { + Symbols []StreamContextSymbol + Reason string + Emitted uint32 + TotalCandidates uint32 + Err string +} + +// StreamContextPosition is the cursor rectangle the daemon ranks around. +// Byte-offset semantics match LIP's `OwnedRange` — 0-based lines and +// chars. Pass a zero-width range at the cursor, or a whole-file range +// (`start_line=0, end_line=lineCount`) for file-level context. +type StreamContextPosition struct { + StartLine int `json:"start_line"` + StartChar int `json:"start_char"` + EndLine int `json:"end_line"` + EndChar int `json:"end_char"` +} + +// streamContextMaxFrames caps how many SymbolInfo frames we accept before +// bailing out — defence against a runaway daemon. Large indexes could +// theoretically produce 10k+ candidates; a hard cap of 1024 is well above +// any realistic caller budget and cheap to enforce. +const streamContextMaxFrames = 1024 + +// StreamContext opens a dedicated connection, sends a `stream_context` +// request, and reads SymbolInfo frames until the daemon writes the +// `end_stream` terminator. Returns (nil, nil) when the daemon is +// unavailable — callers must treat nil as "LIP unavailable" (same contract +// as the rest of the package). +// +// The dedicated connection is intentional: `stream_context` on the +// shared subscriber channel would interleave with IndexStatus pings and +// IndexChanged pushes and complicate parsing. One connection per call is +// fine — the ranking itself dominates latency, and callers shouldn't +// issue this RPC more than a few times per second. +func StreamContext(fileURI string, pos StreamContextPosition, maxTokens uint32, model string) (*StreamContextResult, error) { + conn, err := net.DialTimeout("unix", SocketPath(), 500*time.Millisecond) + if err != nil { + return nil, nil + } + defer conn.Close() + // Overall deadline: the daemon's relevance ranking is heuristic and + // bounded, but pathological inputs could stall. 10 s is generous; for + // a token_budget of ~2000 it completes in ~200 ms typically. + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + + req := map[string]any{ + "type": "stream_context", + "file_uri": fileURI, + "cursor_position": pos, + "max_tokens": maxTokens, + } + if model != "" { + req["model"] = model + } + if err := writeFrame(conn, req); err != nil { + return nil, nil + } + + out := &StreamContextResult{Symbols: make([]StreamContextSymbol, 0, 16)} + for range streamContextMaxFrames + 1 { + frame, err := readFrame(conn) + if err != nil { + if err == io.EOF { + return out, nil + } + return nil, nil + } + // ServerResponse { ok: ServerMessage, error: Option } + inner := frame + if raw, ok := frame["ok"]; ok && len(raw) > 0 && string(raw) != "null" { + _ = json.Unmarshal(raw, &inner) + } + var kind string + _ = json.Unmarshal(inner["type"], &kind) + + switch kind { + case "symbol_info": + var sym struct { + SymbolInfo struct { + URI string `json:"uri"` + DisplayName string `json:"display_name"` + Kind string `json:"kind"` + } `json:"symbol_info"` + RelevanceScore float32 `json:"relevance_score"` + TokenCost uint32 `json:"token_cost"` + } + if b, ok := marshalInner(inner); ok { + _ = json.Unmarshal(b, &sym) + } + out.Symbols = append(out.Symbols, StreamContextSymbol{ + URI: sym.SymbolInfo.URI, + DisplayName: sym.SymbolInfo.DisplayName, + Kind: sym.SymbolInfo.Kind, + RelevanceScore: sym.RelevanceScore, + TokenCost: sym.TokenCost, + }) + case "end_stream": + var end struct { + Reason string `json:"reason"` + Emitted uint32 `json:"emitted"` + TotalCandidates uint32 `json:"total_candidates"` + Error *string `json:"error"` + } + if b, ok := marshalInner(inner); ok { + _ = json.Unmarshal(b, &end) + } + out.Reason = end.Reason + out.Emitted = end.Emitted + out.TotalCandidates = end.TotalCandidates + if end.Error != nil { + out.Err = *end.Error + } + return out, nil + case "error", "unknown_message": + // Daemon rejected the request — treat as unavailable. + return nil, nil + default: + // Unknown frame type mid-stream: skip rather than fail hard. + } + } + return out, nil +} diff --git a/internal/lip/stream_context_test.go b/internal/lip/stream_context_test.go new file mode 100644 index 00000000..33ad24e2 --- /dev/null +++ b/internal/lip/stream_context_test.go @@ -0,0 +1,143 @@ +package lip + +import ( + "encoding/binary" + "encoding/json" + "io" + "net" + "os" + "path/filepath" + "testing" + "time" +) + +// startStreamContextDaemon spins up a Unix socket that responds to any +// request with `frames` in order and then closes. Tests inject the full +// frame sequence they want to exercise — the fake is dumb so behaviour +// under malformed input is exercised by the real daemon's tests, not +// CKB's. +func startStreamContextDaemon(t *testing.T, frames []map[string]any) { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "lipstream") + if err != nil { + t.Fatalf("mkdirtemp: %v", err) + } + sock := filepath.Join(dir, "s.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + os.RemoveAll(dir) + t.Fatalf("listen: %v", err) + } + prev := os.Getenv("LIP_SOCKET") + os.Setenv("LIP_SOCKET", sock) + t.Cleanup(func() { + ln.Close() + os.RemoveAll(dir) + os.Setenv("LIP_SOCKET", prev) + }) + + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + // Drain the incoming stream_context request. + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + var lenBuf [4]byte + if _, err := io.ReadFull(conn, lenBuf[:]); err != nil { + return + } + reqLen := binary.BigEndian.Uint32(lenBuf[:]) + if _, err := io.CopyN(io.Discard, conn, int64(reqLen)); err != nil { + return + } + for _, f := range frames { + b, _ := json.Marshal(f) + var out [4]byte + binary.BigEndian.PutUint32(out[:], uint32(len(b))) + if _, err := conn.Write(out[:]); err != nil { + return + } + if _, err := conn.Write(b); err != nil { + return + } + } + }() +} + +func TestStreamContext_ParsesSymbolInfoThenEndStream(t *testing.T) { + startStreamContextDaemon(t, []map[string]any{ + { + "type": "symbol_info", + "symbol_info": map[string]any{ + "uri": "file:///repo/foo.go", + "display_name": "Foo", + "kind": "function", + }, + "relevance_score": 0.8, + "token_cost": 120, + }, + { + "type": "symbol_info", + "symbol_info": map[string]any{ + "uri": "file:///repo/bar.go", + "display_name": "Bar", + "kind": "struct", + }, + "relevance_score": 0.6, + "token_cost": 80, + }, + { + "type": "end_stream", + "reason": "token_budget", + "emitted": 2, + "total_candidates": 17, + }, + }) + + res, err := StreamContext("file:///repo/foo.go", StreamContextPosition{EndLine: 100}, 1024, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if res == nil { + t.Fatal("nil result, want 2 symbols") + } + if len(res.Symbols) != 2 { + t.Fatalf("symbols = %d, want 2", len(res.Symbols)) + } + if res.Symbols[0].DisplayName != "Foo" || res.Symbols[0].RelevanceScore != 0.8 { + t.Errorf("symbol[0] = %+v", res.Symbols[0]) + } + if res.Reason != "token_budget" || res.Emitted != 2 || res.TotalCandidates != 17 { + t.Errorf("terminator mismatch: %+v", res) + } +} + +func TestStreamContext_DaemonUnavailableReturnsNil(t *testing.T) { + prev := os.Getenv("LIP_SOCKET") + os.Setenv("LIP_SOCKET", "/tmp/ckb-lip-stream-nonexistent.sock") + t.Cleanup(func() { os.Setenv("LIP_SOCKET", prev) }) + + res, err := StreamContext("file:///repo/foo.go", StreamContextPosition{}, 1024, "") + if err != nil { + t.Fatalf("err = %v, want nil (silent degradation contract)", err) + } + if res != nil { + t.Fatalf("res = %+v, want nil", res) + } +} + +func TestStreamContext_ErrorFrameAborts(t *testing.T) { + startStreamContextDaemon(t, []map[string]any{ + { + "type": "error", + "message": "cursor out of range", + "code": "cursor_out_of_range", + }, + }) + res, _ := StreamContext("file:///repo/foo.go", StreamContextPosition{EndLine: 9999}, 1024, "") + if res != nil { + t.Fatalf("res = %+v, want nil on error frame", res) + } +} diff --git a/internal/query/engine.go b/internal/query/engine.go index 18e1bbe6..6a02f0f3 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -68,10 +68,17 @@ type Engine struct { // connection open and receives `index_changed` pushes plus per-ping health // snapshots. `lipHealthCheckedAt` is zero until the first frame arrives — // callers check it before trusting the flags. + // + // `lipSupported` is the set of `type` tags the daemon advertised in its + // handshake. It gates calls to newer RPCs (StreamContext, ExplainMatch, + // ...) on clients talking to an older daemon, instead of letting them + // dispatch and get back an UnknownMessage. Empty when the handshake has + // not yet completed or the daemon predates `supported_messages`. lipHealthMu sync.RWMutex cachedLipMixed bool cachedLipAvailable bool lipHealthCheckedAt time.Time + lipSupported map[string]struct{} lipSubCancel context.CancelFunc // Cache stats diff --git a/internal/query/lip_health.go b/internal/query/lip_health.go index 14c82f67..c9475665 100644 --- a/internal/query/lip_health.go +++ b/internal/query/lip_health.go @@ -26,6 +26,25 @@ func (e *Engine) lipSemanticAvailable() bool { return e.cachedLipAvailable && !e.cachedLipMixed } +// lipSupports reports whether the connected LIP daemon advertised support +// for a given `type` tag in its handshake. Used to gate calls to v2.0+ +// RPCs (ExplainMatch, StreamContext, ...) so we don't dispatch messages +// an older daemon will reject as UnknownMessage. +// +// Returns false when the handshake has not completed yet OR the daemon +// predates `supported_messages` (v1.5). Callers should treat false as +// "fall back to the legacy path" rather than as a hard error — the feature +// is advisory, not guaranteed. +func (e *Engine) lipSupports(msgType string) bool { + e.lipHealthMu.RLock() + defer e.lipHealthMu.RUnlock() + if e.lipSupported == nil { + return false + } + _, ok := e.lipSupported[msgType] + return ok +} + // engineLipSubscriber is the adapter between `lip.Subscribe` and the // Engine's cached health flags. A dedicated type (instead of binding methods // to Engine) keeps the handler interface invisible to the rest of the @@ -51,8 +70,40 @@ func (s engineLipSubscriber) OnIndexChanged(_ lip.IndexChangedEvent) { // no-op-safe — if the daemon is absent, Subscribe backs off and retries // until Close is called. The first health frame lands within `pingInterval` // of daemon availability. +// +// Before starting the subscriber we probe `Handshake` once: it's the only +// RPC that returns `supported_messages`, which we need to gate v2.0+ calls +// on older daemons. Handshake failures are non-fatal (the daemon is likely +// down; Subscribe will retry), and the resulting empty `lipSupported` map +// makes `lipSupports` return false everywhere — callers then fall back to +// their legacy paths. func (e *Engine) startLipSubscriber() { + e.probeHandshake() + ctx, cancel := context.WithCancel(context.Background()) e.lipSubCancel = cancel go lip.Subscribe(ctx, engineLipSubscriber{e: e}) } + +// probeHandshake runs the one-shot handshake and stashes the result on the +// Engine. Split out so tests and re-connect paths can call it. +func (e *Engine) probeHandshake() { + info, _ := lip.Handshake("ckb") + if info == nil { + return + } + supported := make(map[string]struct{}, len(info.SupportedMessages)) + for _, m := range info.SupportedMessages { + supported[m] = struct{}{} + } + e.lipHealthMu.Lock() + e.lipSupported = supported + e.lipHealthMu.Unlock() + if e.logger != nil { + e.logger.Info("LIP handshake", + "daemon_version", info.DaemonVersion, + "protocol_version", info.ProtocolVersion, + "supported_count", len(info.SupportedMessages), + ) + } +} diff --git a/internal/query/lip_ranker.go b/internal/query/lip_ranker.go index e7d33bf8..072730ec 100644 --- a/internal/query/lip_ranker.go +++ b/internal/query/lip_ranker.go @@ -5,6 +5,7 @@ import ( "math" "path/filepath" "sort" + "strings" "github.com/SimplyLiz/CodeMCP/internal/lip" ) @@ -220,6 +221,33 @@ func cosine(v []float32, centroid []float64) float64 { // // Returns nil (not an error) when LIP is unavailable or returns no results. func SemanticSearchWithLIP(query string, topK int, filter string, minScore float32, fn func(fileURIs []string) map[string][]SearchResultItem) []SearchResultItem { + return semanticSearchWithLIP(query, topK, filter, minScore, false, fn) +} + +// SemanticSearchWithLIPExplained is SemanticSearchWithLIP plus evidence +// attachment: for the top `explainTopK` hits, it calls LIP's `explain_match` +// RPC and attaches the returned chunks to every SearchResultItem sourced +// from that file. Makes semantic hits auditable — the caller can surface +// "matched at lines 42–60 with cosine 0.71" instead of a bare file URL. +// +// Gate this on `Engine.lipSupports("explain_match")` — older daemons will +// return UnknownMessage, which `ExplainMatch` silently swallows but costs +// a round-trip per hit regardless. +func SemanticSearchWithLIPExplained(query string, topK int, filter string, minScore float32, fn func(fileURIs []string) map[string][]SearchResultItem) []SearchResultItem { + return semanticSearchWithLIP(query, topK, filter, minScore, true, fn) +} + +// explainTopK bounds how many semantic hits get a follow-up `explain_match` +// round-trip. Explanation is pure overhead for hits the caller never reads, +// so we only explain the top few — matching the realistic display budget. +const explainTopK = 5 + +// explainChunkLines caps the lines-per-chunk the daemon returns. 40 is +// enough to carry a function body or small block without bloating the +// response. +const explainChunkLines = 40 + +func semanticSearchWithLIP(query string, topK int, filter string, minScore float32, explain bool, fn func(fileURIs []string) map[string][]SearchResultItem) []SearchResultItem { hits, _ := lip.NearestByTextFiltered(query, topK, filter, minScore, "") if len(hits) == 0 { return nil @@ -232,6 +260,29 @@ func SemanticSearchWithLIP(query string, topK int, filter string, minScore float } byURI := fn(uris) + // Optionally fetch explanations for the top hits in parallel — one RPC + // per URI, but bounded by explainTopK and only when requested. + evidence := map[string][]SemanticEvidenceChunk{} + if explain { + limit := min(len(hits), explainTopK) + for i := 0; i < limit; i++ { + chunks, _, _ := lip.ExplainMatch(query, hits[i].URI, 2, explainChunkLines, "") + if len(chunks) == 0 { + continue + } + out := make([]SemanticEvidenceChunk, len(chunks)) + for j, c := range chunks { + out[j] = SemanticEvidenceChunk{ + StartLine: c.StartLine, + EndLine: c.EndLine, + Text: c.ChunkText, + Score: c.Score, + } + } + evidence[hits[i].URI] = out + } + } + seen := make(map[string]struct{}, topK*4) var out []SearchResultItem for _, h := range hits { @@ -246,6 +297,9 @@ func SemanticSearchWithLIP(query string, topK int, filter string, minScore float seen[id] = struct{}{} // Blend LIP score into result Score so downstream re-ranking has a signal. item.Score = float64(h.Score) + if chunks, ok := evidence[h.URI]; ok { + item.SemanticEvidence = chunks + } out = append(out, item) } } @@ -254,9 +308,25 @@ func SemanticSearchWithLIP(query string, topK int, filter string, minScore float // lipFileURI returns the file:// URI for a result's source file, suitable for // LIP embedding requests. Returns "" when the result has no location. +// +// Handles three input shapes for `FileId`: +// - repo-relative path (the common case): joined onto repoRoot. +// - absolute filesystem path: used as-is, repoRoot ignored. +// - already a `file://` URI: returned unchanged. +// +// Backends today return relative paths, but the SearchResultItem contract does +// not forbid the other two shapes — this guard keeps a misbehaving backend +// from producing malformed URIs like `file:///repo//abs/path`. func lipFileURI(repoRoot string, r SearchResultItem) string { if r.Location == nil || r.Location.FileId == "" { return "" } - return "file://" + filepath.Join(repoRoot, r.Location.FileId) + id := r.Location.FileId + if strings.HasPrefix(id, "file://") { + return id + } + if filepath.IsAbs(id) { + return "file://" + id + } + return "file://" + filepath.Join(repoRoot, id) } diff --git a/internal/query/lip_stream_context.go b/internal/query/lip_stream_context.go new file mode 100644 index 00000000..ba0425a4 --- /dev/null +++ b/internal/query/lip_stream_context.go @@ -0,0 +1,59 @@ +package query + +import ( + "github.com/SimplyLiz/CodeMCP/internal/lip" +) + +// streamContextMaxTokens caps the total prompt-token budget we ask LIP to +// rank within. 2048 is roughly two screens of code — enough to orient the +// LLM without dominating a typical context window. +const streamContextMaxTokens uint32 = 2048 + +// streamContextLimit bounds how many related symbols we retain in the +// response. The daemon ranks by relevance, so truncating here drops the +// long tail instead of promising more than the caller will use. +const streamContextLimit = 10 + +// relatedViaStreamContext asks LIP for symbols semantically related to the +// whole file (cursor_position spanning line 0 → lineCount). The RPC is +// v2.1+, so we gate on the handshake — older daemons would answer with +// UnknownMessage and every ExplainFile call would eat a round-trip for +// nothing. +// +// Returns an empty slice (not nil) when the daemon is unavailable, the +// feature is unsupported, or the stream yielded nothing — callers append +// to the ExplainFileFacts.Related field and a nil-vs-empty distinction +// isn't meaningful there. +func (e *Engine) relatedViaStreamContext(relPath string, lineCount int) []ExplainFileRelated { + if !e.lipSemanticAvailable() || !e.lipSupports("stream_context") { + return nil + } + if lineCount <= 0 { + lineCount = 1 + } + uri := "file://" + e.repoRoot + "/" + relPath + + res, _ := lip.StreamContext(uri, lip.StreamContextPosition{ + StartLine: 0, + EndLine: lineCount, + }, streamContextMaxTokens, "") + if res == nil || len(res.Symbols) == 0 { + return nil + } + + limit := len(res.Symbols) + if limit > streamContextLimit { + limit = streamContextLimit + } + out := make([]ExplainFileRelated, 0, limit) + for _, s := range res.Symbols[:limit] { + out = append(out, ExplainFileRelated{ + URI: s.URI, + DisplayName: s.DisplayName, + Kind: s.Kind, + Relevance: s.RelevanceScore, + TokenCost: s.TokenCost, + }) + } + return out +} diff --git a/internal/query/navigation.go b/internal/query/navigation.go index 889fc714..8b8d0c44 100644 --- a/internal/query/navigation.go +++ b/internal/query/navigation.go @@ -799,10 +799,24 @@ type ExplainFileFacts struct { Imports []string `json:"imports"` // Key imports Exports []string `json:"exports"` // Key exports/public symbols Hotspots []ExplainFileHotspot `json:"hotspots"` // Local hotspots + Related []ExplainFileRelated `json:"related,omitempty"` Confidence float64 `json:"confidence"` Basis []ConfidenceBasisItem `json:"confidenceBasis"` } +// ExplainFileRelated is a semantically-related symbol surfaced by LIP's +// `stream_context` RPC. The ranking is relevance to the whole file as a +// cursor region, token-budgeted so callers can feed it straight into an +// LLM prompt. Returns an empty list when the daemon is unavailable or +// predates v2.1. +type ExplainFileRelated struct { + URI string `json:"uri"` + DisplayName string `json:"displayName"` + Kind string `json:"kind"` + Relevance float32 `json:"relevance"` + TokenCost uint32 `json:"tokenCost"` +} + // ExplainFileSymbol represents a symbol defined in the file. type ExplainFileSymbol struct { StableId string `json:"stableId"` @@ -952,6 +966,18 @@ func (e *Engine) ExplainFile(ctx context.Context, opts ExplainFileOptions) (*Exp }) } + // Pull related symbols via LIP's stream_context RPC. Ranked by the + // daemon's semantic relevance to the whole file, token-budgeted so a + // caller can paste the list straight into an LLM prompt. Silent no-op + // when the daemon is unavailable or older than v2.1. + related := e.relatedViaStreamContext(relPath, lineCount) + if len(related) > 0 { + confidenceBasis = append(confidenceBasis, ConfidenceBasisItem{ + Backend: "lip", + Status: "available", + }) + } + // Compute confidence based on available backends confidence := computeExplainFileConfidence(confidenceBasis) @@ -983,6 +1009,7 @@ func (e *Engine) ExplainFile(ctx context.Context, opts ExplainFileOptions) (*Exp Imports: imports, Exports: exports, Hotspots: hotspots, + Related: related, Confidence: confidence, Basis: confidenceBasis, }, diff --git a/internal/query/query_expansion.go b/internal/query/query_expansion.go new file mode 100644 index 00000000..468ce8f5 --- /dev/null +++ b/internal/query/query_expansion.go @@ -0,0 +1,66 @@ +package query + +import ( + "strings" + + "github.com/SimplyLiz/CodeMCP/internal/lip" +) + +// maxExpansionTerms caps how many LIP-suggested terms we append to the +// user's query. Beyond ~5 the FTS5 result set starts diluting — the expansion +// is meant to rescue misses, not to replace the original ranking. +const maxExpansionTerms = 5 + +// expandQueryViaLIP returns `query` with up to `maxExpansionTerms` related +// terms appended, fetched from LIP's query_expansion RPC. Only fires for +// short queries (≤ 2 tokens) — longer queries already carry enough lexical +// signal that expansion tends to hurt precision without helping recall. +// +// Silently falls back to the raw query when: +// - the query is empty or already compound, +// - the daemon is unavailable, mixed-models, or predates v1.6, +// - expansion returns no terms. +// +// Expansion is lexical (additional search tokens), so the mixed-models +// gate is relevant: a query embedding produced against one model that then +// matches vocabulary indexed under another will return garbage. We re-use +// `lipSemanticAvailable` as the gate rather than introducing a second one. +func (e *Engine) expandQueryViaLIP(query string) string { + q := strings.TrimSpace(query) + if q == "" { + return query + } + if strings.Count(q, " ") >= 2 { + return query + } + if !e.lipSemanticAvailable() { + return query + } + // Gate on handshake: older daemons that lack query_expansion would + // respond with UnknownMessage. Engines that haven't completed the + // handshake yet fall through unchanged. + if !e.lipSupports("query_expansion") { + return query + } + + terms, _ := lip.QueryExpansion(q, maxExpansionTerms) + if len(terms) == 0 { + return query + } + + // Drop terms that duplicate the original query (case-insensitive) so we + // don't bias FTS ranking by repeating the input. + qLower := strings.ToLower(q) + kept := make([]string, 0, len(terms)) + for _, t := range terms { + t = strings.TrimSpace(t) + if t == "" || strings.ToLower(t) == qLower { + continue + } + kept = append(kept, t) + } + if len(kept) == 0 { + return query + } + return q + " " + strings.Join(kept, " ") +} diff --git a/internal/query/symbols.go b/internal/query/symbols.go index b50e900f..fbd5e27e 100644 --- a/internal/query/symbols.go +++ b/internal/query/symbols.go @@ -317,6 +317,20 @@ type SearchResultItem struct { Lines int `json:"lines,omitempty"` // body line count Cyclomatic int `json:"cyclomatic,omitempty"` // cyclomatic complexity Cognitive int `json:"cognitive,omitempty"` // cognitive complexity + // Evidence attached by the LIP semantic path — ranked chunks from the + // file that best explain why it matched the query. Only populated when + // the result came from SemanticSearchWithLIP with ExplainMatch enabled. + SemanticEvidence []SemanticEvidenceChunk `json:"semanticEvidence,omitempty"` +} + +// SemanticEvidenceChunk is a single ranked snippet returned by LIP's +// `explain_match` RPC, pinning a semantic hit to the specific lines that +// drove the score. Makes semantic results auditable instead of opaque. +type SemanticEvidenceChunk struct { + StartLine uint32 `json:"startLine"` + EndLine uint32 `json:"endLine"` + Text string `json:"text,omitempty"` + Score float32 `json:"score"` } // generateCacheKey creates a deterministic cache key for search options. @@ -389,7 +403,12 @@ func (e *Engine) SearchSymbols(ctx context.Context, opts SearchSymbolsOptions) ( if len(opts.ExcludePatterns) > 0 || opts.MinLines > 0 || opts.MinComplexity > 0 { ftsMultiplier = 10 } - ftsResults, ftsErr := e.SearchSymbolsFTS(ctx, opts.Query, opts.Limit*ftsMultiplier) + // Expand short queries via LIP before hitting FTS — "auth" broadens to + // "auth authenticate authorization principal..." which lifts recall on + // vocabulary-mismatch misses. Falls back to the raw query when LIP is + // unavailable or returns nothing. + ftsQuery := e.expandQueryViaLIP(opts.Query) + ftsResults, ftsErr := e.SearchSymbolsFTS(ctx, ftsQuery, opts.Limit*ftsMultiplier) if ftsErr == nil && len(ftsResults) > 0 { for _, r := range ftsResults { // Skip symbols with no name — they can match via documentation/signature @@ -516,7 +535,14 @@ func (e *Engine) SearchSymbols(ctx context.Context, opts SearchSymbolsOptions) ( const lipFallbackThreshold = 3 if len(results) < lipFallbackThreshold && e.lipSemanticAvailable() { lipSymLimit := opts.Limit * 3 - lipResults := SemanticSearchWithLIP(opts.Query, 20, "", 0, func(fileURIs []string) map[string][]SearchResultItem { + // Attach explain_match evidence when the daemon supports it + // (v2.0+). Gives the caller specific matching chunks per hit + // instead of a bare file URL. + search := SemanticSearchWithLIP + if e.lipSupports("explain_match") { + search = SemanticSearchWithLIPExplained + } + lipResults := search(opts.Query, 20, "", 0, func(fileURIs []string) map[string][]SearchResultItem { // Convert file:// URIs back to repo-relative paths for the batch query. relPaths := make([]string, len(fileURIs)) for i, uri := range fileURIs {