fix(search): never render a backend error or truncation as "no matches"

src/hooks/grep-direct.ts

@@ -338,6 +338,11 @@ export async function handleGrepDirect(
     }
   }
 
+  // On a backend error this throws — by design. The pre-tool-use hook's outer
+  // catch then falls through to the sandboxed VFS shell (deeplake-shell.js),
+  // whose grep-interceptor re-attempts and signals a true backend failure as
+  // grep exit-code 2 + stderr (never a silent "(no matches)"). Swallowing the
+  // error here would pre-empt that retry and the honest signal.
   const output = await grepBothTables(
     api, table, sessionsTable, matchParams, params.targetPath, queryEmbedding,
   );

src/hooks/pre-tool-use.ts

@@ -457,6 +457,13 @@ export async function processPreToolUse(input: PreToolUseInput, deps: ClaudePreT
         // `readVirtualPathContents` (fix #1). Other paths fall back to the
         // same helper which returns null when neither table has a row, at
         // which point we let the shell bundle handle the miss below.
+        //
+        // A genuine backend failure now THROWS out of readVirtualPathContent
+        // (it no longer collapses to null → a misleading "No such file"). We
+        // deliberately let that throw propagate to the outer catch, which
+        // falls through to the sandboxed VFS shell (deeplake-shell.js) whose
+        // readFileBuffer re-attempts and surfaces a real error — preserving
+        // the retry instead of short-circuiting it here.
         content = await readVirtualPathContentFn(api, table, sessionsTable, virtualPath);
       }
       if (content !== null) {
@@ -504,6 +511,8 @@ export async function processPreToolUse(input: PreToolUseInput, deps: ClaudePreT
     if (lsDir) {
       const dir = lsDir.replace(/\/+$/, "") || "/";
       logFn(`direct ls: ${dir}`);
+      // A backend failure throws here; like the read path, we let it propagate
+      // to the outer catch → VFS shell fallback rather than masking it.
       const rows = await listVirtualPathRowsFn(api, table, sessionsTable, dir);
       const entries = new Map<string, { isDir: boolean; size: number }>();
       const prefix = dir === "/" ? "/" : dir + "/";

src/hooks/virtual-table-query.ts

@@ -128,12 +128,21 @@ async function queryUnionRows(
   const unionQuery = buildUnionQuery(memoryQuery, sessionsQuery);
   try {
     return await api.query(unionQuery);
-  } catch {
-    const [memoryRows, sessionRows] = await Promise.all([
-      api.query(memoryQuery).catch(() => []),
-      api.query(sessionsQuery).catch(() => []),
+  } catch (unionErr) {
+    // The dual-table UNION can fail on SQL-compat grounds while the simpler
+    // single-table queries succeed — that is a legitimate fallback. But if
+    // BOTH fallbacks also fail, the backend genuinely could not be queried;
+    // swallowing that to [] would make a backend error look like an empty
+    // result (and "No such file or directory" to the agent). Surface it.
+    const settled = await Promise.allSettled([
+      api.query(memoryQuery),
+      api.query(sessionsQuery),
     ]);
-    return [...memoryRows, ...sessionRows];
+    const fulfilled = settled.filter(
+      (r): r is PromiseFulfilledResult<Row[]> => r.status === "fulfilled",
+    );
+    if (fulfilled.length === 0) throw unionErr;
+    return fulfilled.flatMap((r) => r.value);
   }
 }
 

src/mcp/server.ts

@@ -21,7 +21,7 @@ import { loadConfig } from "../config.js";
 import { DeeplakeApi } from "../deeplake-api.js";
 import { isMissingTableError } from "../deeplake-schema.js";
 import { sqlStr, sqlLike } from "../utils/sql.js";
-import { searchDeeplakeTables, buildGrepSearchOptions, normalizeContent, type GrepMatchParams } from "../shell/grep-core.js";
+import { searchDeeplakeTables, buildGrepSearchOptions, normalizeContent, TRUNCATION_NOTICE, type GrepMatchParams } from "../shell/grep-core.js";
 import { getVersion } from "../cli/version.js";
 
 interface ServerContext {
@@ -89,12 +89,16 @@ server.registerTool(
     opts.limit = limit ?? 10;
 
     try {
-      const rows = await searchDeeplakeTables(ctx.api, ctx.memoryTable, ctx.sessionsTable, opts);
+      const meta = { truncated: false };
+      const rows = await searchDeeplakeTables(ctx.api, ctx.memoryTable, ctx.sessionsTable, opts, meta);
       if (rows.length === 0) return errorResult(`No matches for "${query}".`);
       const lines = rows.map(r => {
         const body = normalizeContent(r.path, r.content);
         return `[${r.path}]\n${body.slice(0, 600)}`;
       });
+      // Tell the caller when the row cap was hit so it doesn't treat a capped
+      // page as the complete set (consistent with the grep path).
+      if (meta.truncated) lines.push(TRUNCATION_NOTICE);
       return { content: [{ type: "text", text: lines.join("\n\n---\n\n") }] };
     } catch (err: unknown) {
       const msg = err instanceof Error ? err.message : String(err);

src/shell/grep-core.ts

@@ -288,6 +288,13 @@ export async function searchDeeplakeTables(
   memoryTable: string,
   sessionsTable: string,
   opts: SearchOptions,
+  /**
+   * Optional out-param. Set `truncated` to true when a per-source row cap was
+   * hit, so callers can warn the agent that matches were dropped (the result
+   * is incomplete, not the full set). Especially important for the regex-only
+   * content scan, which inspects only the first `limit` unordered rows.
+   */
+  meta?: { truncated: boolean },
 ): Promise<ContentRow[]> {
   const { pathFilter, contentScanOnly, likeOp, escapedPattern, prefilterPattern, prefilterPatterns, queryEmbedding, multiWordPatterns } = opts;
   const limit = opts.limit ?? 100;
@@ -371,6 +378,8 @@ export async function searchDeeplakeTables(
       `) AS combined ORDER BY score DESC LIMIT ${outerLimit}`
     );
 
+    if (meta && rows.length >= outerLimit) meta.truncated = true;
+
     const seen = new Set<string>();
     const unique: ContentRow[] = [];
     for (const row of rows) {
@@ -398,6 +407,19 @@ export async function searchDeeplakeTables(
     `) AS combined ORDER BY path, source_order, creation_date`
   );
 
+  // Each subquery is capped at `limit`. If a source returned exactly `limit`
+  // rows it (almost certainly) had more — flag the result as truncated so the
+  // caller can tell the agent it is incomplete.
+  if (meta) {
+    let memCount = 0;
+    let sessCount = 0;
+    for (const row of rows) {
+      if (Number(row["source_order"]) === 0) memCount++;
+      else sessCount++;
+    }
+    if (memCount >= limit || sessCount >= limit) meta.truncated = true;
+  }
+
   return rows.map(row => ({
     path: String(row["path"]),
     content: String(row["content"] ?? ""),
@@ -610,10 +632,11 @@ export async function grepBothTables(
   targetPath: string,
   queryEmbedding?: number[] | null,
 ): Promise<string[]> {
+  const meta = { truncated: false };
   const rows = await searchDeeplakeTables(api, memoryTable, sessionsTable, {
     ...buildGrepSearchOptions(params, targetPath),
     queryEmbedding,
-  });
+  }, meta);
   // Defensive path dedup — memory and sessions tables use disjoint path
   // prefixes in every schema we ship (/summaries/… vs /sessions/…), so the
   // overlap is theoretical, but we dedupe to match grep-interceptor.ts and
@@ -638,9 +661,26 @@ export async function grepBothTables(
           if (trimmed) lines.push(`${r.path}:${line}`);
         }
       }
-      return lines;
+      return withTruncationNotice(lines, meta.truncated);
     }
   }
 
-  return refineGrepMatches(normalized, params);
+  return withTruncationNotice(refineGrepMatches(normalized, params), meta.truncated);
+}
+
+/**
+ * Append an explicit incomplete-results notice when a per-source row cap was
+ * hit. Emitted even when no lines matched: in regex content-scan mode only the
+ * first `limit` rows are fetched, so an empty refined result on a truncated
+ * fetch means "your match may be in the rows we didn't scan" — NOT a confirmed
+ * zero. Collapsing that back to "(no matches)" would reintroduce the exact
+ * silent failure this change exists to remove.
+ */
+export const TRUNCATION_NOTICE =
+  "[hivemind: results incomplete — a per-source row cap was hit, so more matches " +
+  "likely exist. Narrow the path or use a more specific pattern to see them.]";
+
+export function withTruncationNotice(lines: string[], truncated: boolean): string[] {
+  if (!truncated) return lines;
+  return lines.length > 0 ? [...lines, TRUNCATION_NOTICE] : [TRUNCATION_NOTICE];
 }

src/shell/grep-interceptor.ts

@@ -13,6 +13,7 @@ import {
   searchDeeplakeTables,
   normalizeContent,
   refineGrepMatches,
+  withTruncationNotice,
   type GrepMatchParams,
   type ContentRow,
 } from "./grep-core.js";
@@ -120,6 +121,11 @@ export function createGrepCommand(
     }
 
     let rows: ContentRow[] = [];
+    // Remember a backend failure so we can distinguish "the search could not
+    // run" from "the search ran and matched nothing". Cleared as soon as any
+    // query (or the fallback) yields data.
+    let backendError: Error | null = null;
+    const meta = { truncated: false };
     try {
       const searchOptions = {
         ...buildGrepSearchOptions(matchParams, targets[0] ?? ctx.cwd),
@@ -128,11 +134,12 @@ export function createGrepCommand(
         queryEmbedding,
       };
       const queryRows = await Promise.race([
-        searchDeeplakeTables(client, table, sessionsTable ?? "sessions", searchOptions),
+        searchDeeplakeTables(client, table, sessionsTable ?? "sessions", searchOptions, meta),
         new Promise<never>((_, reject) => setTimeout(() => reject(new Error("timeout")), 3000)),
       ]);
       rows.push(...queryRows);
-    } catch {
+    } catch (e) {
+      backendError = e instanceof Error ? e : new Error(String(e));
       rows = []; // fall through to in-memory fallback
     }
 
@@ -147,11 +154,13 @@ export function createGrepCommand(
           limit: 100,
         };
         const lexicalRows = await Promise.race([
-          searchDeeplakeTables(client, table, sessionsTable ?? "sessions", lexicalOptions),
+          searchDeeplakeTables(client, table, sessionsTable ?? "sessions", lexicalOptions, meta),
           new Promise<never>((_, reject) => setTimeout(() => reject(new Error("timeout")), 3000)),
         ]);
         rows.push(...lexicalRows);
-      } catch {
+        if (lexicalRows.length > 0) backendError = null;
+      } catch (e) {
+        backendError = e instanceof Error ? e : new Error(String(e));
         // fall through to in-memory fallback below
       }
     }
@@ -173,6 +182,9 @@ export function createGrepCommand(
         const content = await fs.readFile(fp).catch(() => null);
         if (content !== null) rows.push({ path: fp, content });
       }
+      // The fallback produced data → the earlier backend error is no longer a
+      // user-visible failure.
+      if (rows.length > 0) backendError = null;
     }
 
     // Normalize session JSON blobs to per-turn lines.
@@ -195,10 +207,21 @@ export function createGrepCommand(
       output = refineGrepMatches(normalized, matchParams);
     }
 
-    return {
-      stdout: output.length > 0 ? output.join("\n") + "\n" : "",
-      stderr: "",
-      exitCode: output.length > 0 ? 0 : 1,
-    };
+    if (output.length > 0) {
+      const withNotice = withTruncationNotice(output, meta.truncated);
+      return { stdout: withNotice.join("\n") + "\n", stderr: "", exitCode: 0 };
+    }
+    // No output. Distinguish a genuine zero-match (exit 1) from a search that
+    // could not run (exit 2 + stderr, grep's error convention) so the caller
+    // never mistakes a backend failure for "nothing here".
+    if (backendError) {
+      return {
+        stdout: "",
+        stderr: `grep: hivemind search error: ${backendError.message} ` +
+          `(backend unavailable — result is NOT a confirmed empty match)\n`,
+        exitCode: 2,
+      };
+    }
+    return { stdout: "", stderr: "", exitCode: 1 };
   });
 }

tests/claude-code/grep-core.test.ts

@@ -728,6 +728,39 @@ describe("searchDeeplakeTables", () => {
     const sql = api.query.mock.calls[0][0] as string;
     expect(sql).toContain("LIMIT 100");
   });
+
+  it("flags meta.truncated when the lexical branch fills a per-source cap", async () => {
+    const rows = Array.from({ length: 5 }, (_, i) => ({ path: `/summaries/s${i}`, content: "x", source_order: 0 }));
+    const api = { query: vi.fn().mockResolvedValue(rows) } as any;
+    const meta = { truncated: false };
+    await searchDeeplakeTables(api, "m", "s", {
+      pathFilter: "", contentScanOnly: false, likeOp: "ILIKE", escapedPattern: "x", limit: 5,
+    }, meta);
+    expect(meta.truncated).toBe(true);
+  });
+
+  it("flags meta.truncated when the semantic branch hits the outer cap", async () => {
+    // outerLimit = semanticLimit + lexicalLimit = 20 + 20 = 40 by default.
+    const rows = Array.from({ length: 40 }, (_, i) => ({ path: `/s${i}`, content: "x", score: 0.5 }));
+    const api = { query: vi.fn().mockResolvedValue(rows) } as any;
+    const meta = { truncated: false };
+    await searchDeeplakeTables(api, "m", "s", {
+      pathFilter: "", contentScanOnly: false, likeOp: "ILIKE", escapedPattern: "x",
+      queryEmbedding: [0.1, 0.2, 0.3],
+    }, meta);
+    expect(meta.truncated).toBe(true);
+  });
+
+  it("does NOT flag meta.truncated when the semantic branch is under the cap", async () => {
+    const rows = Array.from({ length: 3 }, (_, i) => ({ path: `/s${i}`, content: "x", score: 0.5 }));
+    const api = { query: vi.fn().mockResolvedValue(rows) } as any;
+    const meta = { truncated: false };
+    await searchDeeplakeTables(api, "m", "s", {
+      pathFilter: "", contentScanOnly: false, likeOp: "ILIKE", escapedPattern: "x",
+      queryEmbedding: [0.1, 0.2, 0.3],
+    }, meta);
+    expect(meta.truncated).toBe(false);
+  });
 });
 
 // ── grepBothTables (end-to-end convenience wrapper) ─────────────────────────

tests/claude-code/grep-direct.test.ts

@@ -15,6 +15,7 @@ vi.mock("../../src/embeddings/client.js", () => ({
 }));
 
 import { parseBashGrep, handleGrepDirect, type GrepParams } from "../../src/hooks/grep-direct.js";
+import { TRUNCATION_NOTICE } from "../../src/shell/grep-core.js";
 
 describe("handleGrepDirect", () => {
   const baseParams: GrepParams = {
@@ -68,6 +69,76 @@ describe("handleGrepDirect", () => {
   });
 });
 
+// ── Honest failure signaling: backend errors propagate (do NOT become empty) ─
+//
+// The core defect behind "hivemind search is silently broken": a backend
+// failure must never read as a genuine zero-match. The fast path intentionally
+// lets the error throw — the pre-tool-use hook's outer catch then falls back to
+// the sandboxed VFS shell (deeplake-shell.js), whose grep-interceptor signals a
+// true backend failure as grep exit-code 2 + stderr (see grep-interceptor.test).
+// What must NOT happen here is the error being swallowed into "(no matches)".
+describe("handleGrepDirect: backend errors are not swallowed", () => {
+  const baseParams: GrepParams = {
+    pattern: "foo", targetPath: "/",
+    ignoreCase: false, wordMatch: false, filesOnly: false, countOnly: false,
+    lineNumber: false, invertMatch: false, fixedString: false,
+  };
+
+  it("propagates the backend error instead of returning '(no matches)'", async () => {
+    const api = { query: vi.fn().mockRejectedValue(new Error("deeplake 500")) } as any;
+    await expect(
+      handleGrepDirect(api, "memory", "sessions", baseParams),
+    ).rejects.toThrow(/500/);
+  });
+});
+
+// ── Truncation signaling ────────────────────────────────────────────────────
+//
+// Each table is fetched with a per-source LIMIT (100). When that cap is hit,
+// matches beyond it are dropped with no signal — so an incomplete result reads
+// to the agent as the complete set. (The regex-only content scan is the worst
+// case: it fetches up to 100 *unordered* rows and regexes them in-memory.)
+// Best practice: never silently truncate — tell the caller the result may be
+// incomplete so it can refine the pattern or narrow the path.
+describe("handleGrepDirect: truncation signaling", () => {
+  const baseParams: GrepParams = {
+    pattern: "foo", targetPath: "/",
+    ignoreCase: false, wordMatch: false, filesOnly: false, countOnly: false,
+    lineNumber: false, invertMatch: false, fixedString: false,
+  };
+  function mockApi(rows: unknown[]) {
+    return { query: vi.fn().mockImplementationOnce(async () => rows) } as any;
+  }
+
+  it("appends the exact incomplete-results notice when a source hits the row cap", async () => {
+    const rows = Array.from({ length: 100 }, (_, i) => ({
+      path: `/summaries/s${i}.md`, content: "foo match", source_order: 0,
+    }));
+    const api = { query: vi.fn().mockResolvedValueOnce(rows) } as any;
+    const r = await handleGrepDirect(api, "memory", "sessions", baseParams);
+    expect(r).toContain(TRUNCATION_NOTICE);
+  });
+
+  it("emits the notice (not '(no matches)') when a truncated scan matched nothing in-window", async () => {
+    // Regex content-scan mode: 100 rows fetched (cap hit, truncated) but none
+    // match the in-memory regex — the real hit may be in rows we didn't scan.
+    // This must NOT read as a confirmed zero-match.
+    const rows = Array.from({ length: 100 }, (_, i) => ({
+      path: `/summaries/s${i}.md`, content: "unrelated body", source_order: 0,
+    }));
+    const api = { query: vi.fn().mockResolvedValueOnce(rows) } as any;
+    const r = await handleGrepDirect(api, "memory", "sessions", { ...baseParams, pattern: "ne+dle" });
+    expect(r).toBe(TRUNCATION_NOTICE);
+    expect(r).not.toBe("(no matches)");
+  });
+
+  it("does NOT add the notice for a normal, fully-returned result", async () => {
+    const api = mockApi([{ path: "/summaries/a.md", content: "foo line", source_order: 0 }]);
+    const r = await handleGrepDirect(api, "memory", "sessions", baseParams);
+    expect(r).not.toContain(TRUNCATION_NOTICE);
+  });
+});
+
 describe("parseBashGrep: long options", () => {
   // Exercises every --long-option handler so the arrow-fn table inside
   // parseBashGrep is fully covered.