feat(executor): SDK 抖动 resume 续跑 + worker 池化评估暂缓 [tsk_74Acz1Nh4K1J]
可选优化任务,先评估收益再做: - worker 池化:评估后暂缓。当前「一 run ↔ 一 pid」模型干净(判活/回收/ re-adopt 全靠每 run 一 pid+心跳,worker 与 DB 隔离 + 重启 re-adopt 天然 崩溃恢复),池化会打破该不变量且只在高并发量下兑现收益。现规模保持 「每任务一进程 + 失败从头重跑」简单模型,仅记入 DESIGN.md。 - SDK resume:实现 worker 内部健壮性小优化。cc.ts 单次会话因流式异常中断 (流断/没收到 result,非超时取消、非 max_turns 终态)且已拿到 sessionId 时, 用 resume 接着原会话续跑一次(保留已干的活,不从头重来),剩余预算不足留 60s。 与「重启 re-adopt / 整体失败 daemon 从头重跑」正交不替代;与模型回退互斥; MAESTRO_SDK_RESUME=0 可关闭。query 改为可注入便于单测。 测试:新增 test/cc.test.ts(6 例:续跑成功/终态不续/无 session 不续/ 开关关闭/超时不续/模型回退)。npm test 全绿 163/163。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+56
-11
@@ -35,21 +35,43 @@ interface AttemptResult {
|
||||
finalText: string;
|
||||
sessionId: string | null;
|
||||
error?: string;
|
||||
/**
|
||||
* 失败是否属于「会话异常中断」——即流式断了 / 没收到 result 就结束(SDK 偶发抖动)。
|
||||
* 这类失败可凭 sessionId 用 resume 续跑(见 runClaude)。区别于:
|
||||
* - 超时/取消(abort,预算已耗尽,续跑无意义)→ false
|
||||
* - 收到 result 但 subtype=error_*(max_turns 等终态,重来只会再撞墙)→ false
|
||||
*/
|
||||
resumable?: boolean;
|
||||
}
|
||||
|
||||
/** runClaude 的 query 注入点:生产用 SDK 的 query,测试传 fake(避免真起 CC)。 */
|
||||
export type CCQuery = typeof query;
|
||||
|
||||
/** resume 续跑至少留这么多预算(避免首攻已耗掉大半 timeout 后续跑没时间干活)。 */
|
||||
const RESUME_MIN_TIMEOUT_MS = 60_000;
|
||||
|
||||
/** 单次 headless CC 会话:流式消息逐行写 transcript,不抛错(失败折叠进 error) */
|
||||
async function attempt(opts: CCOptions, model: string, out: WriteStream): Promise<AttemptResult> {
|
||||
async function attempt(
|
||||
opts: CCOptions,
|
||||
model: string,
|
||||
out: WriteStream,
|
||||
queryImpl: CCQuery,
|
||||
extra?: { resume?: string; timeoutMs?: number },
|
||||
): Promise<AttemptResult> {
|
||||
let sessionId: string | null = null;
|
||||
let finalText = '';
|
||||
let resultOk = false;
|
||||
let resultError: string | undefined;
|
||||
let sawResult = false;
|
||||
let resumable = false;
|
||||
let timedOut = false;
|
||||
|
||||
const abort = new AbortController();
|
||||
const killer = setTimeout(() => abort.abort(new Error('执行超时')), opts.timeoutMs);
|
||||
const timeoutMs = extra?.timeoutMs ?? opts.timeoutMs;
|
||||
const killer = setTimeout(() => { timedOut = true; abort.abort(new Error('执行超时')); }, timeoutMs);
|
||||
|
||||
try {
|
||||
const q = query({
|
||||
const q = queryImpl({
|
||||
prompt: opts.prompt,
|
||||
options: {
|
||||
cwd: opts.cwd,
|
||||
@@ -59,6 +81,7 @@ async function attempt(opts: CCOptions, model: string, out: WriteStream): Promis
|
||||
abortController: abort,
|
||||
model,
|
||||
allowedTools: opts.allowedTools,
|
||||
...(extra?.resume ? { resume: extra.resume } : {}), // 续跑:从该 session 加载历史接着干
|
||||
},
|
||||
});
|
||||
|
||||
@@ -73,44 +96,66 @@ async function attempt(opts: CCOptions, model: string, out: WriteStream): Promis
|
||||
const txt = (message as { result?: unknown }).result;
|
||||
if (typeof txt === 'string') finalText = txt;
|
||||
} else {
|
||||
// 收到了 result(终态错误,如 max_turns)→ 不可 resume(重来只会再撞同一堵墙)
|
||||
const errs = 'errors' in message && Array.isArray(message.errors) ? message.errors.join('; ') : '';
|
||||
resultError = `Claude Code 结束于 ${message.subtype}${errs ? `:${errs}` : ''}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!sawResult && !resultError) resultError = '未收到 result 消息(会话异常结束)';
|
||||
if (!sawResult && !resultError) {
|
||||
// 流跑完了却没见 result:会话异常结束 → 可 resume 续跑
|
||||
resultError = '未收到 result 消息(会话异常结束)';
|
||||
resumable = true;
|
||||
}
|
||||
} catch (e) {
|
||||
// 流式中途抛错:超时/取消导致的 abort 不可 resume;其余(网络抖动/传输错)可 resume
|
||||
resultError = `Claude Code 执行异常:${(e as Error).message}`;
|
||||
resultOk = false;
|
||||
resumable = !timedOut;
|
||||
} finally {
|
||||
clearTimeout(killer);
|
||||
}
|
||||
|
||||
if (!resultOk) return { ok: false, finalText: '', sessionId, error: resultError ?? '未知错误' };
|
||||
if (!resultOk) return { ok: false, finalText: '', sessionId, error: resultError ?? '未知错误', resumable };
|
||||
return { ok: true, finalText, sessionId };
|
||||
}
|
||||
|
||||
/**
|
||||
* 跑一次 headless CC(runner / reviewer 公用)。
|
||||
* 模型可用性兜底:失败且错误信息像模型不可用(not_found/invalid/permission 等)时,
|
||||
* 自动用回退链取一个 ≠ 原模型的模型在同一 run 内重试一次;回退记入 transcript 与 finalText。
|
||||
* 跑一次 headless CC(runner / reviewer 公用)。两层「同一 run 内重试一次」的兜底(互斥,按需触发):
|
||||
*
|
||||
* 1. **SDK 抖动 → resume 续跑**:首攻因会话异常中断(流断/没收到 result,非超时/取消、非 max_turns 终态)
|
||||
* 且已拿到 sessionId 时,用 `resume` 接着原会话续跑一次(保留已干的活,不从头重来)。
|
||||
* 这是 worker 进程内部的健壮性小优化,与「重启靠 re-adopt、失败靠 daemon 从头重跑」正交、不替代它们。
|
||||
* `MAESTRO_SDK_RESUME=0` 可关闭。
|
||||
* 2. **模型不可用 → 回退**:错误信息像模型不可用(not_found/invalid/permission 等)时,
|
||||
* 用回退链取一个 ≠ 原模型的模型重试一次;回退记入 transcript 与 finalText。
|
||||
*
|
||||
* queryImpl 默认用 SDK 的 query,测试可注入 fake。
|
||||
*/
|
||||
export async function runClaude(opts: CCOptions): Promise<CCResult> {
|
||||
export async function runClaude(opts: CCOptions, queryImpl: CCQuery = query): Promise<CCResult> {
|
||||
const dir = transcriptDir();
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const transcriptRef = join(dir, `${opts.runId}.jsonl`);
|
||||
const out = createWriteStream(transcriptRef, { flags: 'a' });
|
||||
const t0 = Date.now();
|
||||
|
||||
try {
|
||||
let r = await attempt(opts, opts.model, out);
|
||||
let r = await attempt(opts, opts.model, out, queryImpl);
|
||||
let modelUsed = opts.model;
|
||||
let fellBack = false;
|
||||
|
||||
// 1. SDK 抖动续跑:会话异常中断 + 有 sessionId + 非模型错误(那归回退处理) + 未关闭
|
||||
if (!r.ok && r.resumable && r.sessionId && !isModelError(r.error ?? '') && process.env.MAESTRO_SDK_RESUME !== '0') {
|
||||
out.write(`${JSON.stringify({ type: 'maestro.session_resume', sessionId: r.sessionId, reason: r.error })}\n`);
|
||||
const remaining = Math.max(opts.timeoutMs - (Date.now() - t0), RESUME_MIN_TIMEOUT_MS);
|
||||
r = await attempt(opts, opts.model, out, queryImpl, { resume: r.sessionId, timeoutMs: remaining });
|
||||
}
|
||||
|
||||
if (!r.ok && r.error && isModelError(r.error)) {
|
||||
const fb = pickFallbackModel(opts.model);
|
||||
if (fb) {
|
||||
out.write(`${JSON.stringify({ type: 'maestro.model_fallback', from: opts.model, to: fb, reason: r.error })}\n`);
|
||||
r = await attempt(opts, fb, out);
|
||||
r = await attempt(opts, fb, out, queryImpl);
|
||||
modelUsed = fb;
|
||||
fellBack = true;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user