diff --git a/.env.example b/.env.example index d4e4e5b..899eb5b 100644 --- a/.env.example +++ b/.env.example @@ -31,8 +31,11 @@ ANTHROPIC_API_KEY=sk-ant-xxxxxxxx # ANTHROPIC_BASE_URL=https://api.anthropic.com # 项目目录（通常无需配置 — 默认为本仓库的父目录，即 process.cwd() 的上一级） # DEFAULT_WORK_DIR=/root/dev -# 单次执行超时时间(秒) +# 单步空闲超时(秒)：某步骤长时间无 SDK 消息活动时 abort，不限制总执行时长 CLAUDE_TIMEOUT=300 +# 工具执行硬顶(秒)：单个工具持续执行超过此时长才视为挂死。默认 900，高于 SDK 单条 Bash 上限(600s) +# 防止长工具(编译/批处理/克隆大仓)被 idle 超时误杀 +CLAUDE_TOOL_TIMEOUT=900 # Codex reviewer 开关 (pipeline review 阶段使用 OpenAI Codex，默认关闭) # CODEX_ENABLED=false diff --git a/src/__tests__/config.test.ts b/src/__tests__/config.test.ts index 1d74e3d..fb3696a 100644 --- a/src/__tests__/config.test.ts +++ b/src/__tests__/config.test.ts @@ -43,6 +43,12 @@ describe('config', () => { const { config } = await loadConfig(); expect(config.claude.timeoutSeconds).toBe(600); }); + + it('should parse CLAUDE_TOOL_TIMEOUT as integer', async () => { + vi.stubEnv('CLAUDE_TOOL_TIMEOUT', '1200'); + const { config } = await loadConfig(); + expect(config.claude.toolTimeoutSeconds).toBe(1200); + }); }); describe('defaults', () => { diff --git a/src/claude/__tests__/executor.test.ts b/src/claude/__tests__/executor.test.ts index 5a7de48..789d0e3 100644 --- a/src/claude/__tests__/executor.test.ts +++ b/src/claude/__tests__/executor.test.ts @@ -12,7 +12,7 @@ vi.mock('node:fs', () => ({ vi.mock('../../config.js', () => ({ config: { - claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' }, + claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, toolTimeoutSeconds: 900, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' }, repoCache: { dir: '/repos/cache' }, workspace: { baseDir: '/tmp/workspaces' }, feishu: { tools: { enabled: false, doc: true, wiki: true, drive: true, bitable: true } }, @@ -70,7 +70,7 @@ vi.mock('../../cron/init.js', () => ({ getCronScheduler: vi.fn(() => null), })); -import { ClaudeExecutor, buildWorkspaceSystemPrompt, classifyCompactOutcome } from '../executor.js'; +import { ClaudeExecutor, buildWorkspaceSystemPrompt, classifyCompactOutcome, shouldExtendIdleTimer } from '../executor.js'; // ============================================================ // Helpers @@ -705,4 +705,103 @@ describe('ClaudeExecutor', () => { expect(r.error).toBeTruthy(); }); }); + + // ============================================================ + // idle 超时 + 工具执行中延展（回归：idle timeout 误杀长工具） + // ============================================================ + describe('idle timeout & in-flight tool extension', () => { + /** + * Mock 一个迭代器：先吐 `pre` 里的消息，之后在下一次拉取时挂起， + * 直到 query 的 abortController 触发才 reject（模拟「长工具执行期间无 SDK 消息， + * 最终被 abort 终止」的真实 SDK 行为）。 + */ + function setupHangingAfter(pre: Array>) { + let idx = 0; + mockQueryInstance[Symbol.asyncIterator].mockReturnValue({ + next: () => { + if (idx < pre.length) { + return Promise.resolve({ value: pre[idx++], done: false }); + } + return new Promise((_resolve, reject) => { + const ac = mockQuery.mock.calls[0]?.[0]?.options?.abortController as AbortController | undefined; + if (ac?.signal?.aborted) { reject(new Error('Operation aborted')); return; } + ac?.signal?.addEventListener('abort', () => reject(new Error('Operation aborted'))); + }); + }, + }); + } + + const INIT_MSG = { type: 'system', subtype: 'init', session_id: 'sess-1', model: 'claude', tools: [] }; + const ASSISTANT_TOOL_USE = { + type: 'assistant', + message: { content: [{ type: 'tool_use', name: 'Bash', id: 'tool-1', input: { command: 'sleep 9999' } }] }, + }; + + it('extends idle timer while a tool is in-flight, aborting only after the tool hard cap', async () => { + vi.useFakeTimers(); + try { + // idle 窗口 1s，工具硬顶 3s + setupHangingAfter([INIT_MSG, ASSISTANT_TOOL_USE]); + const p = executor.execute(makeInput({ timeoutSeconds: 1, toolTimeoutSeconds: 3 })); + + // 越过第一个 idle 窗口(1s)：工具仍在执行 → 应延展，不应 abort + await vi.advanceTimersByTimeAsync(1500); + const ac = mockQuery.mock.calls[0][0].options.abortController as AbortController; + expect(ac.signal.aborted).toBe(false); + + // 越过工具硬顶(3s)：判定挂死并 abort + await vi.advanceTimersByTimeAsync(2000); + const result = await p; + + expect(result.success).toBe(false); + expect(ac.signal.aborted).toBe(true); + expect(result.error).toMatch(/tool execution timeout/i); + expect(result.error).toMatch(/longer than 3s/); + } finally { + vi.useRealTimers(); + } + }); + + it('aborts at the idle window when no tool is in-flight (model idle)', async () => { + vi.useFakeTimers(); + try { + // 只有 init，之后挂起且没有任何工具在执行 → 纯模型空闲 + setupHangingAfter([INIT_MSG]); + const p = executor.execute(makeInput({ timeoutSeconds: 1, toolTimeoutSeconds: 30 })); + + await vi.advanceTimersByTimeAsync(1200); + const result = await p; + + const ac = mockQuery.mock.calls[0][0].options.abortController as AbortController; + expect(ac.signal.aborted).toBe(true); + expect(result.success).toBe(false); + expect(result.error).toMatch(/idle timeout/i); + expect(result.error).not.toMatch(/tool execution timeout/i); + } finally { + vi.useRealTimers(); + } + }); + }); +}); + +// ============================================================ +// shouldExtendIdleTimer — 纯函数判定逻辑 +// ============================================================ +describe('shouldExtendIdleTimer', () => { + it('extends when a tool is in-flight and under the hard cap', () => { + expect(shouldExtendIdleTimer({ inFlightToolCount: 1, toolWaitElapsedMs: 1000, toolTimeoutMs: 900_000 })).toBe(true); + }); + + it('does not extend when no tool is in-flight (model idle)', () => { + expect(shouldExtendIdleTimer({ inFlightToolCount: 0, toolWaitElapsedMs: 1000, toolTimeoutMs: 900_000 })).toBe(false); + }); + + it('does not extend once the tool exceeds the hard cap (stuck tool)', () => { + expect(shouldExtendIdleTimer({ inFlightToolCount: 1, toolWaitElapsedMs: 900_000, toolTimeoutMs: 900_000 })).toBe(false); + expect(shouldExtendIdleTimer({ inFlightToolCount: 2, toolWaitElapsedMs: 900_001, toolTimeoutMs: 900_000 })).toBe(false); + }); + + it('extends for multiple concurrent in-flight tools under the cap', () => { + expect(shouldExtendIdleTimer({ inFlightToolCount: 3, toolWaitElapsedMs: 0, toolTimeoutMs: 1000 })).toBe(true); + }); }); diff --git a/src/claude/__tests__/github-orgs.test.ts b/src/claude/__tests__/github-orgs.test.ts index aa8f9a6..4108e95 100644 --- a/src/claude/__tests__/github-orgs.test.ts +++ b/src/claude/__tests__/github-orgs.test.ts @@ -10,7 +10,7 @@ import { tmpdir } from 'node:os'; vi.mock('../../config.js', () => ({ config: { - claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' }, + claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, toolTimeoutSeconds: 900, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' }, repoCache: { dir: '/repos/cache' }, workspace: { baseDir: '/tmp/workspaces' }, feishu: { tools: { enabled: false, doc: true, wiki: true, drive: true, bitable: true } }, diff --git a/src/claude/executor.ts b/src/claude/executor.ts index d466494..00c502e 100644 --- a/src/claude/executor.ts +++ b/src/claude/executor.ts @@ -205,6 +205,8 @@ export interface ExecuteInput extends ExecuteOptions { systemPromptOverride?: string; /** 覆盖单步空闲超时秒数 (默认使用 CLAUDE_TIMEOUT 配置)。每收到一条 SDK 消息就重置，不限制总时长 */ timeoutSeconds?: number; + /** 覆盖工具执行硬顶秒数 (默认使用 CLAUDE_TOOL_TIMEOUT 配置)。工具持续执行超过此时长才视为挂死 */ + toolTimeoutSeconds?: number; /** 硬性总超时秒数（从开始计时，不因活动重置）。适用于 routing 等必须快速完成的短任务 */ hardTimeoutSeconds?: number; /** 图片附件（多模态输入） */ @@ -608,6 +610,25 @@ export function classifyCompactOutcome(signals: { }; } +/** + * 判定 idle 计时器触发时应「延展」还是「abort」。 + * + * idle 计时器的本意是捕捉「模型/API 停止产出、且没有工具在跑」的真·卡死。 + * 但单条工具调用(如最长可达 600s 的前台 Bash、克隆大仓、编译)执行期间不会有任何 + * SDK 消息到达，会被误判为空闲而 abort（线上表现："idle timeout ... (total elapsed 701s)"）。 + * + * 因此：只要仍有工具在执行(inFlightToolCount>0)且未超过工具硬顶(toolTimeoutMs)， + * 就延展计时而非 abort —— 此时 agent 并未卡死，只是工具耗时长。 + * 超过硬顶仍未返回，才判定为工具挂死并 abort。 + */ +export function shouldExtendIdleTimer(opts: { + inFlightToolCount: number; + toolWaitElapsedMs: number; + toolTimeoutMs: number; +}): boolean { + return opts.inFlightToolCount > 0 && opts.toolWaitElapsedMs < opts.toolTimeoutMs; +} + export class ClaudeExecutor { /** 运行中的 query 实例 (用于 abort) */ private runningQueries = new Map(); @@ -630,25 +651,51 @@ export class ClaudeExecutor { const startTime = Date.now(); const abortController = new AbortController(); const idleTimeoutMs = (input.timeoutSeconds ?? config.claude.timeoutSeconds) * 1000; + // 工具执行硬顶：单个工具(或一组并发工具)持续执行超过此时长才视为挂死。 + // 默认高于 SDK 单条 Bash 上限(600s)，确保正常长工具能跑完。见 shouldExtendIdleTimer。 + const toolTimeoutMs = (input.toolTimeoutSeconds ?? config.claude.toolTimeoutSeconds) * 1000; let timedOut = false; + // 区分两种超时：true=工具执行超硬顶(挂死)，false=模型空闲。用于错误信息与日志。 + let toolStuck = false; // 跟踪是否有过工具活动（canUseTool 被调用过） // 有工具活动说明 agent 在积极工作，API 处理大上下文思考下一步可能需要更长时间 let hasToolActivity = false; + // 当前 in-flight 工具周期的开始时间（null 表示当前没有工具在执行）。 + // 在消息循环里随 pendingToolCalls 的增减维护，用于计算工具已执行时长。 + let toolWaitStartedAt: number | null = null; // 滑动窗口 idle 超时：每收到一条 SDK 消息就重置计时器 // 只在某一步长时间无活动时才 abort，不限制总执行时长 // 当 agent 有过工具活动时，使用 2 倍超时（API 处理大上下文后思考下一步可能较慢） let idleTimer: ReturnType = undefined!; let lastResetSource = 'init'; - const resetIdleTimer = (source?: string) => { + // pendingToolCalls 在下方声明（消息循环作用域），这里用闭包延迟读取其 size 作为 in-flight 信号 + const inFlightToolCount = () => pendingToolCalls.size; + const baseIdleTimeout = () => (hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs); + const resetIdleTimer = (source?: string, overrideMs?: number) => { clearTimeout(idleTimer); if (source) lastResetSource = source; - const effectiveTimeout = hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs; + const effectiveTimeout = overrideMs ?? baseIdleTimeout(); idleTimer = setTimeout(() => { + // 计时器触发：若仍有工具在执行且未超工具硬顶，说明 agent 没卡死(只是工具耗时长)， + // 延展计时而非 abort；超过硬顶仍未返回才判定挂死。 + const inFlight = inFlightToolCount(); + const toolElapsed = toolWaitStartedAt !== null ? Date.now() - toolWaitStartedAt : 0; + if (shouldExtendIdleTimer({ inFlightToolCount: inFlight, toolWaitElapsedMs: toolElapsed, toolTimeoutMs })) { + // 取「常规窗口」与「距硬顶剩余时间」的较小值，使硬顶能较精确地生效 + const nextMs = Math.max(1000, Math.min(baseIdleTimeout(), toolTimeoutMs - toolElapsed)); + logger.debug({ sessionKey, inFlight, toolElapsedMs: toolElapsed, toolTimeoutMs, nextMs }, + 'Idle timer fired but tools in-flight — extending instead of aborting'); + resetIdleTimer('tool-in-flight', nextMs); + return; + } timedOut = true; + toolStuck = inFlight > 0; // 工具仍在执行但已超硬顶 → 挂死 abortController.abort(); - logger.warn({ sessionKey, idleTimeoutMs: effectiveTimeout, hasToolActivity, lastResetSource, elapsedMs: Date.now() - startTime }, - 'Claude query idle timeout — no SDK message received, aborting'); + logger.warn({ sessionKey, effectiveTimeout, hasToolActivity, lastResetSource, toolStuck, inFlight, elapsedMs: Date.now() - startTime }, + toolStuck + ? 'Claude query tool execution timeout — tool exceeded hard cap, aborting' + : 'Claude query idle timeout — no SDK message received, aborting'); }, effectiveTimeout); }; resetIdleTimer(); @@ -1104,6 +1151,12 @@ export class ClaudeExecutor { } } + // 工具开始执行 → 记录 in-flight 周期起点（仅在从「无工具」转为「有工具」时设置）。 + // idle 计时器据此判断工具是否仍在跑，从而延展而非误杀（见 shouldExtendIdleTimer）。 + if (pendingToolCalls.size > 0 && toolWaitStartedAt === null) { + toolWaitStartedAt = Date.now(); + } + // 累积对话轨迹（用于 restart 上下文传递） if (conversationTraceBytes < MAX_TRACE_BYTES && (turnText.length > 0 || traceToolCalls.length > 0)) { const turnTextStr = turnText.join(''); @@ -1181,6 +1234,10 @@ export class ClaudeExecutor { } } } + // 全部 in-flight 工具均已返回 → 结束当前 in-flight 周期，恢复常规 idle 语义 + if (pendingToolCalls.size === 0) { + toolWaitStartedAt = null; + } break; } @@ -1208,7 +1265,9 @@ export class ClaudeExecutor { const errorMsg = timedOut ? (input.hardTimeoutSeconds && durationMs >= input.hardTimeoutSeconds * 1000 ? `Query hard timeout after ${input.hardTimeoutSeconds}s total execution time` - : `Query idle timeout after ${(hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs) / 1000}s with no activity (total elapsed: ${Math.round(durationMs / 1000)}s)`) + : toolStuck + ? `Query tool execution timeout — a tool ran longer than ${toolTimeoutMs / 1000}s without returning (total elapsed: ${Math.round(durationMs / 1000)}s)` + : `Query idle timeout after ${baseIdleTimeout() / 1000}s with no activity (total elapsed: ${Math.round(durationMs / 1000)}s)`) : (err instanceof Error ? err.message : String(err)); // 30MB message size 超限 + resume 模式 → 累积的会话历史太大 diff --git a/src/config.ts b/src/config.ts index d5febe2..003d4a4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,6 +77,12 @@ export const config = { apiBaseUrl: process.env.ANTHROPIC_BASE_URL || '', /** 单步空闲超时 (秒)：某步骤长时间无 SDK 消息活动时 abort。不限制总执行时长 */ timeoutSeconds: parseInt(process.env.CLAUDE_TIMEOUT || '300', 10), + /** + * 工具执行硬顶 (秒)：单个工具(或一组并发工具)持续执行超过此时长才视为挂死并 abort。 + * 默认 900，明显高于 SDK 单条 Bash 上限(600s)，确保正常长工具(编译/批处理/克隆大仓)能跑完， + * 同时兜底真正挂死的工具(如等待 stdin 的命令)。区别于 timeoutSeconds：工具执行期间不算"空闲"。 + */ + toolTimeoutSeconds: parseInt(process.env.CLAUDE_TOOL_TIMEOUT || '900', 10), /** 模型名称，默认 claude-opus-4-8 (Opus 4.8) */ model: process.env.CLAUDE_MODEL || 'claude-opus-4-8', /** thinking 模式: 'adaptive' (自适应思考) | 'disabled' */