Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 135 additions & 1 deletion src/claude/__tests__/executor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

vi.mock('../../config.js', () => ({
config: {
claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '' },
claude: { defaultWorkDir: '/tmp/work', timeoutSeconds: 300, model: 'claude-opus-4-6', thinking: 'adaptive', effort: 'max', maxTurns: 500, maxBudgetUsd: 50, apiBaseUrl: '', firstTokenStallRetries: 2 },
repoCache: { dir: '/repos/cache' },
workspace: { baseDir: '/tmp/workspaces' },
feishu: { tools: { enabled: false, doc: true, wiki: true, drive: true, bitable: true } },
Expand Down Expand Up @@ -583,6 +583,140 @@
});
});

describe('first-token stall auto-retry', () => {
// 回归:收到 system:init 后,首个 assistant token 因网关/API 瞬时抖动迟迟不到,
// 触发空闲超时。此前直接判失败让用户白等 5 分钟;现应自动重试。

/**
* Mock query 工厂:前 `stallCount` 次调用在 init 之后停摆(挂起直到 abort 触发后 reject),
* 之后的调用返回 init + 成功 result。abortController 从每次 query 的 options 读取。
*/
function setupFirstTokenStall(stallCount: number, successResult = 'recovered') {
let callIndex = 0;
mockQuery.mockImplementation((arg: { options?: { abortController?: AbortController } }) => {
callIndex++;
const thisCall = callIndex;
const ac = arg?.options?.abortController;
return {
close: vi.fn(),
[Symbol.asyncIterator]() {
let step = 0;
return {
next() {
if (thisCall <= stallCount) {
if (step === 0) {
step++;
return Promise.resolve({ value: { type: 'system', subtype: 'init', session_id: `s${thisCall}`, model: 'claude', tools: [] }, done: false });
}
// 停摆:挂起,直到 abort 触发后 reject(模拟 SDK abort 抛出)
return new Promise((_, reject) => {
if (ac?.signal?.aborted) { reject(new Error('Operation aborted')); return; }
ac?.signal?.addEventListener('abort', () => reject(new Error('Operation aborted')), { once: true });
});
}
if (step === 0) { step++; return Promise.resolve({ value: { type: 'system', subtype: 'init', session_id: `s${thisCall}`, model: 'claude', tools: [] }, done: false }); }
if (step === 1) { step++; return Promise.resolve({ value: { type: 'result', subtype: 'success', session_id: `s${thisCall}`, result: successResult, duration_ms: 10 }, done: false }); }
return Promise.resolve({ value: undefined, done: true });
},
};
},
};
});
}

it('retries the query when it stalls after init and succeeds on retry', async () => {
vi.useFakeTimers();
try {
setupFirstTokenStall(1); // stall once, then succeed
const promise = executor.execute(makeInput({ timeoutSeconds: 1 }));
// 驱动:首次 idle 超时(1s) → abort → 退避 → 重试 → 成功
await vi.runAllTimersAsync();
const result = await promise;

expect(result.success).toBe(true);
expect(result.output).toBe('recovered');
// 初次 + 1 次重试 = 2 次 query
expect(mockQuery).toHaveBeenCalledTimes(2);
} finally {
vi.useRealTimers();
}
});

it('gives up after firstTokenStallRetries and returns the idle timeout error', async () => {
vi.useFakeTimers();
try {
setupFirstTokenStall(99); // 始终停摆
const promise = executor.execute(makeInput({ timeoutSeconds: 1 }));
await vi.runAllTimersAsync();
const result = await promise;

expect(result.success).toBe(false);
expect(result.error).toMatch(/idle timeout/i);
// 初次 + 2 次重试(config.firstTokenStallRetries=2)= 3 次 query
expect(mockQuery).toHaveBeenCalledTimes(3);
} finally {
vi.useRealTimers();
}
});

it('does NOT retry when idle timeout fires after the first token has arrived', async () => {
vi.useFakeTimers();
try {
mockQuery.mockImplementation((arg: { options?: { abortController?: AbortController } }) => {
const ac = arg?.options?.abortController;
return {
close: vi.fn(),
[Symbol.asyncIterator]() {
let step = 0;
return {
next() {
if (step === 0) { step++; return Promise.resolve({ value: { type: 'system', subtype: 'init', session_id: 's1', model: 'claude', tools: [] }, done: false }); }
// 首个 assistant token 已到达 → firstTokenSeen=true
if (step === 1) { step++; return Promise.resolve({ value: { type: 'assistant', message: { content: [{ type: 'text', text: 'working...' }] } }, done: false }); }
// 之后停摆(mid-execution idle,非首字停摆)
return new Promise((_, reject) => {
if (ac?.signal?.aborted) { reject(new Error('Operation aborted')); return; }
ac?.signal?.addEventListener('abort', () => reject(new Error('Operation aborted')), { once: true });
});
},
};
},
};
});

const promise = executor.execute(makeInput({ timeoutSeconds: 1 }));
await vi.runAllTimersAsync();
const result = await promise;

expect(result.success).toBe(false);
expect(result.error).toMatch(/idle timeout/i);
// 首个 token 已到达,属普通空闲超时,不重试
expect(mockQuery).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});

it('does not retry when stall retries are disabled (firstTokenStallRetries=0)', async () => {
const { config } = await import('../../config.js');
const prev = config.claude.firstTokenStallRetries;
config.claude.firstTokenStallRetries = 0;
vi.useFakeTimers();
try {
setupFirstTokenStall(99);
const promise = executor.execute(makeInput({ timeoutSeconds: 1 }));
await vi.runAllTimersAsync();
const result = await promise;

expect(result.success).toBe(false);
expect(mockQuery).toHaveBeenCalledTimes(1); // 关闭重试 → 仅一次
} finally {
vi.useRealTimers();
config.claude.firstTokenStallRetries = prev;
}
});
});

describe('buildWorkspaceSystemPrompt — platform constraints', () => {
it('includes platform execution constraints section', () => {
const prompt = buildWorkspaceSystemPrompt('/tmp/work');
Expand All @@ -601,2 +735,2 @@
});
});
39 changes: 37 additions & 2 deletions src/claude/executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ const WRITE_TOOLS = new Set([
'Edit', 'Write', 'NotebookEdit', 'Bash', 'Skill',
]);

/** 首字停摆重试前的退避时长(ms),给瞬时过载的网关一点恢复时间 */
const FIRST_TOKEN_STALL_BACKOFF_MS = 1500;

/** 源仓库保护拦截提示 */
const SOURCE_REPO_DENY_MSG = '源仓库保护:目标位于 DEFAULT_WORK_DIR 下的源仓库,禁止直接修改。请使用 setup_workspace 工具创建隔离工作区后再修改。';

Expand Down Expand Up @@ -243,6 +246,8 @@ export interface ExecuteInput extends ExecuteOptions {
inplaceEdit?: boolean;
/** AskUserQuestion 回调:拦截 AskUserQuestion 工具调用,由上层实现交互式卡片 */
onAskUser?: (questions: Array<{ question: string; header?: string; options: Array<{ label: string; description?: string }>; multiSelect?: boolean }>) => Promise<Record<string, string>>;
/** 内部:已进行的首字停摆重试次数(递归重试时累加,调用方不应设置) */
_firstTokenStallRetries?: number;
}

/** 扫描 defaultWorkDir 下的 git 项目名列表(best-effort) */
Expand Down Expand Up @@ -604,9 +609,15 @@ export class ClaudeExecutor {
const abortController = new AbortController();
const idleTimeoutMs = (input.timeoutSeconds ?? config.claude.timeoutSeconds) * 1000;
let timedOut = false;
// 区分超时类型:idle(空闲超时,可能因首字停摆触发,可重试)vs hard(硬性总超时,不重试)
let timeoutKind: 'idle' | 'hard' | undefined;
// 跟踪是否有过工具活动(canUseTool 被调用过)
// 有工具活动说明 agent 在积极工作,API 处理大上下文思考下一步可能需要更长时间
let hasToolActivity = false;
// 跟踪是否已收到首个 assistant token / 流式片段 / 工具活动。
// 仅在 system:init 之后、首个 token 之前停摆时(firstTokenSeen 仍为 false),
// 空闲超时才被判定为"首字停摆"并触发自动重试。
let firstTokenSeen = false;

// 滑动窗口 idle 超时:每收到一条 SDK 消息就重置计时器
// 只在某一步长时间无活动时才 abort,不限制总执行时长
Expand All @@ -619,8 +630,9 @@ export class ClaudeExecutor {
const effectiveTimeout = hasToolActivity ? idleTimeoutMs * 2 : idleTimeoutMs;
idleTimer = setTimeout(() => {
timedOut = true;
timeoutKind = 'idle';
abortController.abort();
logger.warn({ sessionKey, idleTimeoutMs: effectiveTimeout, hasToolActivity, lastResetSource, elapsedMs: Date.now() - startTime },
logger.warn({ sessionKey, idleTimeoutMs: effectiveTimeout, hasToolActivity, firstTokenSeen, lastResetSource, elapsedMs: Date.now() - startTime },
'Claude query idle timeout — no SDK message received, aborting');
}, effectiveTimeout);
};
Expand All @@ -632,6 +644,7 @@ export class ClaudeExecutor {
const hardTimeoutMs = input.hardTimeoutSeconds * 1000;
hardTimer = setTimeout(() => {
timedOut = true;
timeoutKind = 'hard';
abortController.abort();
logger.warn({ sessionKey, hardTimeoutMs, elapsedMs: Date.now() - startTime },
'Claude query hard timeout — total execution time exceeded, aborting');
Expand Down Expand Up @@ -830,8 +843,10 @@ export class ClaudeExecutor {
canUseTool: async (toolName: string, inputObj: Record<string, unknown>) => {
// canUseTool 在每次工具执行前触发,说明 agent 仍在活跃工作
// 1. 标记工具活动 → 后续 idle timer 使用 2 倍超时(API 处理大上下文后思考较慢)
// 2. 重置 idle timer → 防止长时间 MCP 工具执行导致误超时
// 2. 标记首个 token 已到达 → 排除首字停摆重试
// 3. 重置 idle timer → 防止长时间 MCP 工具执行导致误超时
hasToolActivity = true;
firstTokenSeen = true;
resetIdleTimer(`canUseTool:${toolName}`);

// workspace 变更后 deny 所有后续工具调用,迫使 agent 只输出文本后自然结束
Expand Down Expand Up @@ -1045,6 +1060,8 @@ export class ClaudeExecutor {
break;

case 'assistant': {
// 收到 assistant 消息说明首个 token 已到达 — 排除首字停摆重试
firstTokenSeen = true;
// 提取文本输出和工具调用
const turnText: string[] = [];
const turnTools: ToolCallInfo[] = [];
Expand Down Expand Up @@ -1162,6 +1179,8 @@ export class ClaudeExecutor {
break messageLoop;

default:
// stream_event(流式片段)说明 token 已开始流动 — 排除首字停摆重试
if (message.type === 'stream_event') firstTokenSeen = true;
// tool_progress, stream_event 等其他消息类型 — 记录以便诊断 idle timeout 间隙
logger.debug({ sessionKey, messageType: message.type, subtype: (message as Record<string, unknown>).subtype },
'SDK message (non-primary)');
Expand Down Expand Up @@ -1200,6 +1219,22 @@ export class ClaudeExecutor {
});
}

// 首字停摆自动重试:收到 system:init 后,首个 assistant token 因网关/API 瞬时抖动
// 迟迟不到而触发空闲超时(firstTokenSeen 仍为 false)。此时无任何 token 产出,
// 重新发起 query 即可(保留 resume,会话状态未推进)。带退避,上限取 config。
const stallRetries = input._firstTokenStallRetries ?? 0;
const maxStallRetries = config.claude.firstTokenStallRetries;
if (timeoutKind === 'idle' && !firstTokenSeen && stallRetries < maxStallRetries) {
logger.warn(
{ sessionKey, attempt: stallRetries + 1, maxRetries: maxStallRetries, elapsedMs: durationMs, backoffMs: FIRST_TOKEN_STALL_BACKOFF_MS },
'First-token stall detected (no assistant token after init) — retrying query',
);
// 关闭已 abort 的旧 query,防止子进程泄漏(abort 后 close 是幂等的)
try { q.close(); } catch { /* already aborted/closed */ }
await new Promise(resolve => setTimeout(resolve, FIRST_TOKEN_STALL_BACKOFF_MS));
return this.execute({ ...input, _firstTokenStallRetries: stallRetries + 1 });
}

logger.error({ sessionKey, err: errorMsg, timedOut }, 'Claude Agent SDK query error');

return {
Expand Down
5 changes: 5 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ export const config = {
apiBaseUrl: process.env.ANTHROPIC_BASE_URL || '',
/** 单步空闲超时 (秒):某步骤长时间无 SDK 消息活动时 abort。不限制总执行时长 */
timeoutSeconds: parseInt(process.env.CLAUDE_TIMEOUT || '300', 10),
/**
* 首字停摆自动重试次数:收到 system:init 后,首个 assistant token 因网关/API 瞬时抖动
* 迟迟不到而触发空闲超时时,自动重新发起 query 的最大次数(默认 2)。设为 0 关闭。
*/
firstTokenStallRetries: parseInt(process.env.CLAUDE_FIRST_TOKEN_STALL_RETRIES || '2', 10),
/** 模型名称,默认 claude-opus-4-8 (Opus 4.8) */
model: process.env.CLAUDE_MODEL || 'claude-opus-4-8',
/** thinking 模式: 'adaptive' (自适应思考) | 'disabled' */
Expand Down
Loading