Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,11 @@ ANTHROPIC_API_KEY=sk-ant-xxxxxxxx
# ANTHROPIC_BASE_URL=https://api.anthropic.com
# 项目目录(通常无需配置 — 默认为本仓库的父目录,即 process.cwd() 的上一级)
# DEFAULT_WORK_DIR=/root/dev
# 单次执行超时时间(秒)
# 单步空闲超时(秒):某步骤长时间无 SDK 消息活动时 abort,不限制总执行时长
CLAUDE_TIMEOUT=300
# 工具执行硬顶(秒):单个工具持续执行超过此时长才视为挂死。默认 900,高于 SDK 单条 Bash 上限(600s)
# 防止长工具(编译/批处理/克隆大仓)被 idle 超时误杀
CLAUDE_TOOL_TIMEOUT=900
# Codex reviewer 开关 (pipeline review 阶段使用 OpenAI Codex,默认关闭)
# CODEX_ENABLED=false

Expand Down
6 changes: 6 additions & 0 deletions src/__tests__/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ describe('config', () => {
const { config } = await loadConfig();
expect(config.claude.timeoutSeconds).toBe(600);
});

it('should parse CLAUDE_TOOL_TIMEOUT as integer', async () => {
vi.stubEnv('CLAUDE_TOOL_TIMEOUT', '1200');
const { config } = await loadConfig();
expect(config.claude.toolTimeoutSeconds).toBe(1200);
});
});

describe('defaults', () => {
Expand Down
103 changes: 101 additions & 2 deletions src/claude/__tests__/executor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ vi.mock('node:fs', () => ({

vi.mock('../../config.js', () => ({
config: {
claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' },
claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, toolTimeoutSeconds: 900, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' },
repoCache: { dir: '/repos/cache' },
workspace: { baseDir: '/tmp/workspaces' },
feishu: { tools: { enabled: false, doc: true, wiki: true, drive: true, bitable: true } },
Expand Down Expand Up @@ -70,7 +70,7 @@ vi.mock('../../cron/init.js', () => ({
getCronScheduler: vi.fn(() => null),
}));

import { ClaudeExecutor, buildWorkspaceSystemPrompt, classifyCompactOutcome } from '../executor.js';
import { ClaudeExecutor, buildWorkspaceSystemPrompt, classifyCompactOutcome, shouldExtendIdleTimer } from '../executor.js';

// ============================================================
// Helpers
Expand Down Expand Up @@ -705,4 +705,103 @@ describe('ClaudeExecutor', () => {
expect(r.error).toBeTruthy();
});
});

// ============================================================
// idle 超时 + 工具执行中延展(回归:idle timeout 误杀长工具)
// ============================================================
describe('idle timeout & in-flight tool extension', () => {
/**
* Mock 一个迭代器:先吐 `pre` 里的消息,之后在下一次拉取时挂起,
* 直到 query 的 abortController 触发才 reject(模拟「长工具执行期间无 SDK 消息,
* 最终被 abort 终止」的真实 SDK 行为)。
*/
function setupHangingAfter(pre: Array<Record<string, unknown>>) {
let idx = 0;
mockQueryInstance[Symbol.asyncIterator].mockReturnValue({
next: () => {
if (idx < pre.length) {
return Promise.resolve({ value: pre[idx++], done: false });
}
return new Promise((_resolve, reject) => {
const ac = mockQuery.mock.calls[0]?.[0]?.options?.abortController as AbortController | undefined;
if (ac?.signal?.aborted) { reject(new Error('Operation aborted')); return; }
ac?.signal?.addEventListener('abort', () => reject(new Error('Operation aborted')));
});
},
});
}

const INIT_MSG = { type: 'system', subtype: 'init', session_id: 'sess-1', model: 'claude', tools: [] };
const ASSISTANT_TOOL_USE = {
type: 'assistant',
message: { content: [{ type: 'tool_use', name: 'Bash', id: 'tool-1', input: { command: 'sleep 9999' } }] },
};

it('extends idle timer while a tool is in-flight, aborting only after the tool hard cap', async () => {
vi.useFakeTimers();
try {
// idle 窗口 1s,工具硬顶 3s
setupHangingAfter([INIT_MSG, ASSISTANT_TOOL_USE]);
const p = executor.execute(makeInput({ timeoutSeconds: 1, toolTimeoutSeconds: 3 }));

// 越过第一个 idle 窗口(1s):工具仍在执行 → 应延展,不应 abort
await vi.advanceTimersByTimeAsync(1500);
const ac = mockQuery.mock.calls[0][0].options.abortController as AbortController;
expect(ac.signal.aborted).toBe(false);

// 越过工具硬顶(3s):判定挂死并 abort
await vi.advanceTimersByTimeAsync(2000);
const result = await p;

expect(result.success).toBe(false);
expect(ac.signal.aborted).toBe(true);
expect(result.error).toMatch(/tool execution timeout/i);
expect(result.error).toMatch(/longer than 3s/);
} finally {
vi.useRealTimers();
}
});

it('aborts at the idle window when no tool is in-flight (model idle)', async () => {
vi.useFakeTimers();
try {
// 只有 init,之后挂起且没有任何工具在执行 → 纯模型空闲
setupHangingAfter([INIT_MSG]);
const p = executor.execute(makeInput({ timeoutSeconds: 1, toolTimeoutSeconds: 30 }));

await vi.advanceTimersByTimeAsync(1200);
const result = await p;

const ac = mockQuery.mock.calls[0][0].options.abortController as AbortController;
expect(ac.signal.aborted).toBe(true);
expect(result.success).toBe(false);
expect(result.error).toMatch(/idle timeout/i);
expect(result.error).not.toMatch(/tool execution timeout/i);
} finally {
vi.useRealTimers();
}
});
});
});

// ============================================================
// shouldExtendIdleTimer — 纯函数判定逻辑
// ============================================================
describe('shouldExtendIdleTimer', () => {
it('extends when a tool is in-flight and under the hard cap', () => {
expect(shouldExtendIdleTimer({ inFlightToolCount: 1, toolWaitElapsedMs: 1000, toolTimeoutMs: 900_000 })).toBe(true);
});

it('does not extend when no tool is in-flight (model idle)', () => {
expect(shouldExtendIdleTimer({ inFlightToolCount: 0, toolWaitElapsedMs: 1000, toolTimeoutMs: 900_000 })).toBe(false);
});

it('does not extend once the tool exceeds the hard cap (stuck tool)', () => {
expect(shouldExtendIdleTimer({ inFlightToolCount: 1, toolWaitElapsedMs: 900_000, toolTimeoutMs: 900_000 })).toBe(false);
expect(shouldExtendIdleTimer({ inFlightToolCount: 2, toolWaitElapsedMs: 900_001, toolTimeoutMs: 900_000 })).toBe(false);
});

it('extends for multiple concurrent in-flight tools under the cap', () => {
expect(shouldExtendIdleTimer({ inFlightToolCount: 3, toolWaitElapsedMs: 0, toolTimeoutMs: 1000 })).toBe(true);
});
});
2 changes: 1 addition & 1 deletion src/claude/__tests__/github-orgs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { tmpdir } from 'node:os';

vi.mock('../../config.js', () => ({
config: {
claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' },
claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, toolTimeoutSeconds: 900, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' },
repoCache: { dir: '/repos/cache' },
workspace: { baseDir: '/tmp/workspaces' },
feishu: { tools: { enabled: false, doc: true, wiki: true, drive: true, bitable: true } },
Expand Down
69 changes: 64 additions & 5 deletions src/claude/executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ export interface ExecuteInput extends ExecuteOptions {
systemPromptOverride?: string;
/** 覆盖单步空闲超时秒数 (默认使用 CLAUDE_TIMEOUT 配置)。每收到一条 SDK 消息就重置,不限制总时长 */
timeoutSeconds?: number;
/** 覆盖工具执行硬顶秒数 (默认使用 CLAUDE_TOOL_TIMEOUT 配置)。工具持续执行超过此时长才视为挂死 */
toolTimeoutSeconds?: number;
/** 硬性总超时秒数(从开始计时,不因活动重置)。适用于 routing 等必须快速完成的短任务 */
hardTimeoutSeconds?: number;
/** 图片附件(多模态输入) */
Expand Down Expand Up @@ -608,6 +610,25 @@ export function classifyCompactOutcome(signals: {
};
}

/**
* 判定 idle 计时器触发时应「延展」还是「abort」。
*
* idle 计时器的本意是捕捉「模型/API 停止产出、且没有工具在跑」的真·卡死。
* 但单条工具调用(如最长可达 600s 的前台 Bash、克隆大仓、编译)执行期间不会有任何
* SDK 消息到达,会被误判为空闲而 abort(线上表现:"idle timeout ... (total elapsed 701s)")。
*
* 因此:只要仍有工具在执行(inFlightToolCount>0)且未超过工具硬顶(toolTimeoutMs),
* 就延展计时而非 abort —— 此时 agent 并未卡死,只是工具耗时长。
* 超过硬顶仍未返回,才判定为工具挂死并 abort。
*/
export function shouldExtendIdleTimer(opts: {
inFlightToolCount: number;
toolWaitElapsedMs: number;
toolTimeoutMs: number;
}): boolean {
return opts.inFlightToolCount > 0 && opts.toolWaitElapsedMs < opts.toolTimeoutMs;
}

export class ClaudeExecutor {
/** 运行中的 query 实例 (用于 abort) */
private runningQueries = new Map<string, Query>();
Expand All @@ -630,25 +651,51 @@ export class ClaudeExecutor {
const startTime = Date.now();
const abortController = new AbortController();
const idleTimeoutMs = (input.timeoutSeconds ?? config.claude.timeoutSeconds) * 1000;
// 工具执行硬顶:单个工具(或一组并发工具)持续执行超过此时长才视为挂死。
// 默认高于 SDK 单条 Bash 上限(600s),确保正常长工具能跑完。见 shouldExtendIdleTimer。
const toolTimeoutMs = (input.toolTimeoutSeconds ?? config.claude.toolTimeoutSeconds) * 1000;
let timedOut = false;
// 区分两种超时:true=工具执行超硬顶(挂死),false=模型空闲。用于错误信息与日志。
let toolStuck = false;
// 跟踪是否有过工具活动(canUseTool 被调用过)
// 有工具活动说明 agent 在积极工作,API 处理大上下文思考下一步可能需要更长时间
let hasToolActivity = false;
// 当前 in-flight 工具周期的开始时间(null 表示当前没有工具在执行)。
// 在消息循环里随 pendingToolCalls 的增减维护,用于计算工具已执行时长。
let toolWaitStartedAt: number | null = null;

// 滑动窗口 idle 超时:每收到一条 SDK 消息就重置计时器
// 只在某一步长时间无活动时才 abort,不限制总执行时长
// 当 agent 有过工具活动时,使用 2 倍超时(API 处理大上下文后思考下一步可能较慢)
let idleTimer: ReturnType<typeof setTimeout> = undefined!;
let lastResetSource = 'init';
const resetIdleTimer = (source?: string) => {
// pendingToolCalls 在下方声明(消息循环作用域),这里用闭包延迟读取其 size 作为 in-flight 信号
const inFlightToolCount = () => pendingToolCalls.size;
const baseIdleTimeout = () => (hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs);
const resetIdleTimer = (source?: string, overrideMs?: number) => {
clearTimeout(idleTimer);
if (source) lastResetSource = source;
const effectiveTimeout = hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs;
const effectiveTimeout = overrideMs ?? baseIdleTimeout();
idleTimer = setTimeout(() => {
// 计时器触发:若仍有工具在执行且未超工具硬顶,说明 agent 没卡死(只是工具耗时长),
// 延展计时而非 abort;超过硬顶仍未返回才判定挂死。
const inFlight = inFlightToolCount();
const toolElapsed = toolWaitStartedAt !== null ? Date.now() - toolWaitStartedAt : 0;
if (shouldExtendIdleTimer({ inFlightToolCount: inFlight, toolWaitElapsedMs: toolElapsed, toolTimeoutMs })) {
// 取「常规窗口」与「距硬顶剩余时间」的较小值,使硬顶能较精确地生效
const nextMs = Math.max(1000, Math.min(baseIdleTimeout(), toolTimeoutMs - toolElapsed));
logger.debug({ sessionKey, inFlight, toolElapsedMs: toolElapsed, toolTimeoutMs, nextMs },
'Idle timer fired but tools in-flight — extending instead of aborting');
resetIdleTimer('tool-in-flight', nextMs);
return;
}
timedOut = true;
toolStuck = inFlight > 0; // 工具仍在执行但已超硬顶 → 挂死
abortController.abort();
logger.warn({ sessionKey, idleTimeoutMs: effectiveTimeout, hasToolActivity, lastResetSource, elapsedMs: Date.now() - startTime },
'Claude query idle timeout — no SDK message received, aborting');
logger.warn({ sessionKey, effectiveTimeout, hasToolActivity, lastResetSource, toolStuck, inFlight, elapsedMs: Date.now() - startTime },
toolStuck
? 'Claude query tool execution timeout — tool exceeded hard cap, aborting'
: 'Claude query idle timeout — no SDK message received, aborting');
}, effectiveTimeout);
};
resetIdleTimer();
Expand Down Expand Up @@ -1104,6 +1151,12 @@ export class ClaudeExecutor {
}
}

// 工具开始执行 → 记录 in-flight 周期起点(仅在从「无工具」转为「有工具」时设置)。
// idle 计时器据此判断工具是否仍在跑,从而延展而非误杀(见 shouldExtendIdleTimer)。
if (pendingToolCalls.size > 0 && toolWaitStartedAt === null) {
toolWaitStartedAt = Date.now();
}

// 累积对话轨迹(用于 restart 上下文传递)
if (conversationTraceBytes < MAX_TRACE_BYTES && (turnText.length > 0 || traceToolCalls.length > 0)) {
const turnTextStr = turnText.join('');
Expand Down Expand Up @@ -1181,6 +1234,10 @@ export class ClaudeExecutor {
}
}
}
// 全部 in-flight 工具均已返回 → 结束当前 in-flight 周期,恢复常规 idle 语义
if (pendingToolCalls.size === 0) {
toolWaitStartedAt = null;
}
break;
}

Expand Down Expand Up @@ -1208,7 +1265,9 @@ export class ClaudeExecutor {
const errorMsg = timedOut
? (input.hardTimeoutSeconds && durationMs >= input.hardTimeoutSeconds * 1000
? `Query hard timeout after ${input.hardTimeoutSeconds}s total execution time`
: `Query idle timeout after ${(hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs) / 1000}s with no activity (total elapsed: ${Math.round(durationMs / 1000)}s)`)
: toolStuck
? `Query tool execution timeout — a tool ran longer than ${toolTimeoutMs / 1000}s without returning (total elapsed: ${Math.round(durationMs / 1000)}s)`
: `Query idle timeout after ${baseIdleTimeout() / 1000}s with no activity (total elapsed: ${Math.round(durationMs / 1000)}s)`)
: (err instanceof Error ? err.message : String(err));

// 30MB message size 超限 + resume 模式 → 累积的会话历史太大
Expand Down
6 changes: 6 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ export const config = {
apiBaseUrl: process.env.ANTHROPIC_BASE_URL || '',
/** 单步空闲超时 (秒):某步骤长时间无 SDK 消息活动时 abort。不限制总执行时长 */
timeoutSeconds: parseInt(process.env.CLAUDE_TIMEOUT || '300', 10),
/**
* 工具执行硬顶 (秒):单个工具(或一组并发工具)持续执行超过此时长才视为挂死并 abort。
* 默认 900,明显高于 SDK 单条 Bash 上限(600s),确保正常长工具(编译/批处理/克隆大仓)能跑完,
* 同时兜底真正挂死的工具(如等待 stdin 的命令)。区别于 timeoutSeconds:工具执行期间不算"空闲"。
*/
toolTimeoutSeconds: parseInt(process.env.CLAUDE_TOOL_TIMEOUT || '900', 10),
/** 模型名称,默认 claude-opus-4-8 (Opus 4.8) */
model: process.env.CLAUDE_MODEL || 'claude-opus-4-8',
/** thinking 模式: 'adaptive' (自适应思考) | 'disabled' */
Expand Down
Loading