fix: add timeout + fail-open to recall search path

kagura-agent · kagura-agent · commit 57f069e69dcc · 2026-04-10T12:16:04.000+08:00
Fixes #1452 — auto-recall can block gateway startup / first-turn path long enough to fail health checks when embedding or LLM calls are slow. Changes: - Add configurable `recall.timeoutMs` (default 10s) - Wrap embedder.embedQuery() with timeout in RecallEngine.search(); falls back to FTS-only results on timeout - Wrap LLM skill relevance judgment with timeout in searchSkills(); falls back to returning all candidates on timeout - Add top-level timeout in memory_search tool handler; returns empty results with `timedOut: true` flag on timeout - All timeouts fail-open: partial/empty results, never throw - Recall exceptions never propagate to gateway top level
diff --git a/apps/memos-local-plugin/src/config.ts b/apps/memos-local-plugin/src/config.ts
@@ -65,6 +65,7 @@ export function resolveConfig(raw: Partial<MemosLocalConfig> | undefined, stateD
       mmrLambda: cfg.recall?.mmrLambda ?? DEFAULTS.mmrLambda,
       recencyHalfLifeDays: cfg.recall?.recencyHalfLifeDays ?? DEFAULTS.recencyHalfLifeDays,
       vectorSearchMaxChunks: cfg.recall?.vectorSearchMaxChunks ?? DEFAULTS.vectorSearchMaxChunks,
+      timeoutMs: cfg.recall?.timeoutMs ?? DEFAULTS.recallTimeoutMs,
     },
     dedup: {
       similarityThreshold: cfg.dedup?.similarityThreshold ?? DEFAULTS.dedupSimilarityThreshold,
diff --git a/apps/memos-local-plugin/src/recall/engine.ts b/apps/memos-local-plugin/src/recall/engine.ts
@@ -9,6 +9,25 @@ import { Summarizer } from "../ingest/providers";
 
 export type SkillSearchScope = "mix" | "self" | "public";
 
+/** Race a promise against a timeout. Returns fallback value on timeout instead of throwing. */
+function withTimeout<T>(promise: Promise<T>, ms: number, fallback: T, label: string, log: { warn: (msg: string, ...args: unknown[]) => void }): Promise<T> {
+  if (ms <= 0) return promise;
+  return new Promise<T>((resolve) => {
+    let settled = false;
+    const timer = setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        log.warn(`recall: ${label} timed out after ${ms}ms — returning fallback`);
+        resolve(fallback);
+      }
+    }, ms);
+    promise.then(
+      (val) => { if (!settled) { settled = true; clearTimeout(timer); resolve(val); } },
+      (err) => { if (!settled) { settled = true; clearTimeout(timer); log.warn(`recall: ${label} failed: ${err}`); resolve(fallback); } },
+    );
+  });
+}
+
 export interface RecallOptions {
   query?: string;
   maxResults?: number;
@@ -48,13 +67,24 @@ export class RecallEngine {
       : [];
 
     let vecCandidates: Array<{ chunkId: string; score: number }> = [];
+    const timeoutMs = this.ctx.config.recall!.timeoutMs ?? 10_000;
     if (query) {
       try {
-        const queryVec = await this.embedder.embedQuery(query);
-        const maxChunks = recallCfg.vectorSearchMaxChunks && recallCfg.vectorSearchMaxChunks > 0
-          ? recallCfg.vectorSearchMaxChunks
-          : undefined;
-        vecCandidates = vectorSearch(this.store, queryVec, candidatePool, maxChunks, ownerFilter);
+        const queryVec = await withTimeout(
+          this.embedder.embedQuery(query),
+          timeoutMs,
+          null,
+          "embedQuery",
+          this.ctx.log,
+        );
+        if (queryVec) {
+          const maxChunks = recallCfg.vectorSearchMaxChunks && recallCfg.vectorSearchMaxChunks > 0
+            ? recallCfg.vectorSearchMaxChunks
+            : undefined;
+          vecCandidates = vectorSearch(this.store, queryVec, candidatePool, maxChunks, ownerFilter);
+        } else {
+          this.ctx.log.warn("Vector search skipped (embedding timed out), using FTS only");
+        }
       } catch (err) {
         this.ctx.log.warn(`Vector search failed, using FTS only: ${err}`);
       }
@@ -101,7 +131,13 @@ export class RecallEngine {
       }
 
       try {
-        const qv = await this.embedder.embedQuery(query).catch(() => null);
+        const qv = await withTimeout(
+          this.embedder.embedQuery(query).catch(() => null),
+          timeoutMs,
+          null,
+          "hubMemEmbedQuery",
+          this.ctx.log,
+        );
         if (qv) {
           const memEmbs = this.store.getVisibleHubMemoryEmbeddings("__hub__");
           const scored: Array<{ id: string; score: number }> = [];
@@ -302,15 +338,24 @@ export class RecallEngine {
 
     // Vector search on description embedding
     let vecCandidates: Array<{ skillId: string; score: number }> = [];
+    const timeoutMs = this.ctx.config.recall!.timeoutMs ?? 10_000;
     try {
-      const queryVec = await this.embedder.embedQuery(query);
-      const allEmb = this.store.getSkillEmbeddings(scope, currentOwner);
-      vecCandidates = allEmb.map((row) => ({
-        skillId: row.skillId,
-        score: cosineSimilarity(queryVec, row.vector),
-      }));
-      vecCandidates.sort((a, b) => b.score - a.score);
-      vecCandidates = vecCandidates.slice(0, TOP_CANDIDATES);
+      const queryVec = await withTimeout(
+        this.embedder.embedQuery(query),
+        timeoutMs,
+        null,
+        "skillEmbedQuery",
+        this.ctx.log,
+      );
+      if (queryVec) {
+        const allEmb = this.store.getSkillEmbeddings(scope, currentOwner);
+        vecCandidates = allEmb.map((row) => ({
+          skillId: row.skillId,
+          score: cosineSimilarity(queryVec, row.vector),
+        }));
+        vecCandidates.sort((a, b) => b.score - a.score);
+        vecCandidates = vecCandidates.slice(0, TOP_CANDIDATES);
+      }
     } catch (err) {
       this.ctx.log.warn(`Skill vector search failed, using FTS only: ${err}`);
     }
@@ -336,9 +381,16 @@ export class RecallEngine {
 
     if (candidateSkills.length === 0) return [];
 
-    // LLM relevance judgment
+    // LLM relevance judgment (with timeout — fail-open returns all candidates)
     const summarizer = new Summarizer(this.ctx.config.summarizer, this.ctx.log, this.ctx.openclawAPI);
-    const relevantIndices = await this.judgeSkillRelevance(summarizer, query, candidateSkills);
+    const allIndices = candidateSkills.map((_, i) => i);
+    const relevantIndices = await withTimeout(
+      this.judgeSkillRelevance(summarizer, query, candidateSkills),
+      timeoutMs,
+      allIndices,
+      "judgeSkillRelevance",
+      this.ctx.log,
+    );
 
     return relevantIndices.map((idx) => {
       const { skill, rrfScore } = candidateSkills[idx];
diff --git a/apps/memos-local-plugin/src/tools/memory-search.ts b/apps/memos-local-plugin/src/tools/memory-search.ts
@@ -25,6 +25,8 @@ function emptyHubResult(scope: HubScope): HubSearchResult {
 }
 
 export function createMemorySearchTool(engine: RecallEngine, store?: SqliteStore, ctx?: PluginContext, sharedState?: { lastSearchTime: number }): ToolDefinition {
+  const EMPTY_RESULT = { hits: [], meta: { usedMinScore: 0, usedMaxResults: 0, totalCandidates: 0, timedOut: true, note: "Search timed out \u2014 returning empty results to avoid blocking the critical path." } };
+
   return {
     name: "memory_search",
     description:
@@ -66,27 +68,48 @@ export function createMemorySearchTool(engine: RecallEngine, store?: SqliteStore
       const minScore = input.minScore as number | undefined;
       const ownerFilter = resolveOwnerFilter(input.owner);
       const scope = resolveScope(input.scope);
+      const timeoutMs = ctx?.config?.recall?.timeoutMs ?? 10_000;
 
-      const localSearch = engine.search({
-        query,
-        maxResults,
-        minScore,
-        ownerFilter,
-      });
+      // Top-level timeout: never block the critical path longer than timeoutMs
+      const doSearch = async () => {
+        const localSearch = engine.search({
+          query,
+          maxResults,
+          minScore,
+          ownerFilter,
+        });
+
+        if (scope === "local" || !store || !ctx) {
+          return localSearch;
+        }
 
-      if (scope === "local" || !store || !ctx) {
-        return localSearch;
-      }
+        const [local, hub] = await Promise.all([
+          localSearch,
+          hubSearchMemories(store, ctx, { query, maxResults, scope, hubAddress: input.hubAddress as string | undefined, userToken: input.userToken as string | undefined }).catch((err) => {
+            ctx.log.warn(`Hub search failed, using local-only results: ${err}`);
+            return emptyHubResult(scope);
+          }),
+        ]);
 
-      const [local, hub] = await Promise.all([
-        localSearch,
-        hubSearchMemories(store, ctx, { query, maxResults, scope, hubAddress: input.hubAddress as string | undefined, userToken: input.userToken as string | undefined }).catch((err) => {
-          ctx.log.warn(`Hub search failed, using local-only results: ${err}`);
-          return emptyHubResult(scope);
-        }),
-      ]);
+        return { local, hub };
+      };
 
-      return { local, hub };
+      if (timeoutMs <= 0) return doSearch();
+
+      return new Promise((resolve) => {
+        let settled = false;
+        const timer = setTimeout(() => {
+          if (!settled) {
+            settled = true;
+            ctx?.log?.warn?.(`memory_search timed out after ${timeoutMs}ms \u2014 returning empty results`);
+            resolve(EMPTY_RESULT);
+          }
+        }, timeoutMs);
+        doSearch().then(
+          (val) => { if (!settled) { settled = true; clearTimeout(timer); resolve(val); } },
+          (err) => { if (!settled) { settled = true; clearTimeout(timer); ctx?.log?.warn?.(`memory_search failed: ${err}`); resolve(EMPTY_RESULT); } },
+        );
+      });
     },
   };
 }
diff --git a/apps/memos-local-plugin/src/types.ts b/apps/memos-local-plugin/src/types.ts
@@ -312,6 +312,8 @@ export interface MemosLocalConfig {
     recencyHalfLifeDays?: number;
     /** Cap vector search to this many most recent chunks. 0 = no cap (search all; may get slower with 200k+ chunks). If you set a cap for performance, use a large value (e.g. 200000–300000) so older memories are still in the window; FTS always searches all. */
     vectorSearchMaxChunks?: number;
+    /** Hard timeout in milliseconds for the entire recall search path. When exceeded, partial results (FTS-only) are returned instead of blocking. 0 = no timeout. Default 10000 (10s). */
+    timeoutMs?: number;
   };
   dedup?: {
     similarityThreshold?: number;
@@ -337,6 +339,7 @@ export const DEFAULTS = {
   mmrLambda: 0.7,
   recencyHalfLifeDays: 14,
   vectorSearchMaxChunks: 0,
+  recallTimeoutMs: 10_000,
   dedupSimilarityThreshold: 0.80,
   evidenceWrapperTag: "STORED_MEMORY",
   excerptMinChars: 200,
