Files
maestro/src/daemon/orchestrator.ts
T
wangjia c89d6d129c feat: 重试与超时策略细化(可配 + 退避)tsk_N5wO3Armums2
## 变更摘要

### 1. 项目级可配重试上限与超时
- `Project` 新增 `maxRetries`(默认 2)和 `timeoutMs`(默认 30min = 1800000ms)字段
- SQLite schema:`projects` 表补 `max_retries` / `timeout_ms` 列(带 DEFAULT 的轻量迁移)
- `createProject` / `patchProject` 支持设置 / 校验新字段(maxRetries>=0, timeoutMs>=1000)
- API `POST /api/projects` 与 `PATCH /api/projects/:id` 透传新字段
- `runner.ts` 将 `project.timeoutMs` 传给 `runClaude`,不再固定 30min

### 2. 失败重试指数退避
- 编排器内存维护 `backoffUntil` Map(taskId → nextRetryAt),daemon 重启后清空
- 退避公式:`min(30s * 2^(attempt-1), 10min)`,第 1 次 30s / 第 2 次 60s / ...
- `claimable()` 过滤退避冷却中的任务
- `nowMs` 注入点(默认 `Date.now`)使测试可快进时钟验证退避行为

### 3. needs_attention 一键重投
- `Task` 新增 `retryBaseline` 字段(默认 0):记录上次重投时的失败 run 基线
- `Store.requeueTask(taskId)`:设置基线 = 当前失败 run 数 → 转 `queued`(仅限 needs_attention)
- 编排器用净失败数(`allFailed - retryBaseline`)判断是否已耗尽重试次数
- API `POST /api/tasks/:id/requeue` + MCP `requeue_task` 工具

### 4. 测试
- 重命名 `MAX_RETRIES` → `DEFAULT_MAX_RETRIES`,新增导出 `computeBackoffMs`
- 新增测试(共 +18):maxRetries=1/0、退避时序(精确 ms 边界)、重投后基线重置
- 迁移测试扩展:验证旧库补列后 maxRetries/timeoutMs/retryBaseline 使用默认值

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-13 04:04:41 +08:00

260 lines
12 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { Store } from '../store/index.js';
import type { Project, Task, ReviewVerdict } from '../model/types.js';
import { rankByScore } from '../model/scoring.js';
import { createWorktree, worktreeDiff, type WorktreeDiff, type WorktreeInfo } from '../executor/worktree.js';
import { runTask, type RunnerFn } from '../executor/runner.js';
import { runVerify, type VerifyFn } from '../executor/verify.js';
import { reviewCode, reviewSecurity, type ReviewerFn } from '../executor/reviewer.js';
import { pickModel } from '../executor/models.js';
/** 失败后默认最多自动重试次数(项目级 maxRetries 未设置时回退此值) */
export const DEFAULT_MAX_RETRIES = 2;
/** 指数退避基础等待(第 n 次重试等待 BASE * 2^(n-1),上限 MAX */
const BACKOFF_BASE_MS = 30_000; // 30s
const BACKOFF_MAX_MS = 10 * 60_000; // 10min
/** 第 attempt 次重试(1-indexed)的等待毫秒 */
export function computeBackoffMs(attempt: number): number {
return Math.min(BACKOFF_BASE_MS * (2 ** (attempt - 1)), BACKOFF_MAX_MS);
}
export interface OrchestratorLogger {
info(msg: string): void;
error(msg: string): void;
}
/** 依赖注入点:测试传 mock,生产用真实现 */
export interface OrchestratorDeps {
runner: RunnerFn;
reviewCode: ReviewerFn; // code reviewkind=reviewer 的 run
reviewSecurity: ReviewerFn; // 安全审计(kind=security 的 run
verify: VerifyFn;
createWorktree: (repoPath: string, taskId: string, baseBranch: string) => Promise<WorktreeInfo>;
worktreeDiff: (repoPath: string, dir: string, branch: string, baseBranch: string) => Promise<WorktreeDiff>;
/** 当前时间戳(ms);测试注入可控时钟(默认 Date.now) */
nowMs: () => number;
}
export interface Orchestrator {
/** 跑一轮领取(同步领取 + 异步执行,不阻塞)。错误只记日志。 */
tick(): void;
/** 等待所有在途执行收尾(测试用) */
drain(): Promise<void>;
/** 在途任务 id(防同任务重复领取) */
readonly inflight: ReadonlyMap<string, string>;
}
/**
* 编排器核心(与定时器解耦,便于测试)。
* 每轮对 status=active 且 autonomy≠manual 的项目:在途数 < concurrency 时领任务——
* ready 叶子(deps 全 doneauto-easy 只领 easy)或 queued(重试/重启遗留)。
* 成功 → setResult + exec_review;失败 → failed → 重试 ≤MAX_RETRIES 次 → needs_attention。
*/
export function createOrchestrator(store: Store, log: OrchestratorLogger, deps: Partial<OrchestratorDeps> = {}): Orchestrator {
const d: OrchestratorDeps = { runner: runTask, reviewCode, reviewSecurity, verify: runVerify, createWorktree, worktreeDiff, nowMs: Date.now, ...deps };
const inflight = new Map<string, string>(); // taskId → projectId
const pending = new Set<Promise<void>>();
/** 退避表:taskId → 下次可领取时间戳(ms);daemon 重启后清空,已有 queued 任务即刻可领 */
const backoffUntil = new Map<string, number>();
/**
* 本项目可领取的任务:queued(重试/孤儿)+ ready 叶子且 deps 全 doneauto-easy 只挑 easy。
* 按调度分降序返回(score = 自身分 + 已完成依赖分 + 等待解锁的 blocked 任务分,见 model/scoring.ts)。
*/
function claimable(project: Project): Array<{ task: Task; score: number }> {
const nowMs = d.nowMs();
const tasks = store.listTasks(project.id);
const byId = new Map(tasks.map((t) => [t.id, t]));
const parents = new Set(tasks.filter((t) => t.parentId).map((t) => t.parentId as string));
const easyOnly = project.autonomy === 'auto-easy';
const candidates = tasks.filter((t) => {
if (inflight.has(t.id)) return false;
if (t.status !== 'ready' && t.status !== 'queued') return false;
if (parents.has(t.id)) return false; // 非叶子(容器)跳过
if (easyOnly && t.complexity !== 'easy') return false;
const until = backoffUntil.get(t.id);
if (until !== undefined && nowMs < until) return false; // 退避冷却中
return t.deps.every((dep) => byId.get(dep)?.status === 'done');
});
return rankByScore(candidates, tasks);
}
/** 跑一个独立复审 runkind=reviewer/security)。任何失败折叠为 summary=失败原因、verdict=null,不抛错、不挡任务。 */
async function runOneReview(
kind: 'reviewer' | 'security',
fn: ReviewerFn,
task: Task,
project: Project,
wt: WorktreeInfo,
executorReport: string,
): Promise<{ summary: string | null; verdict: ReviewVerdict | null }> {
const label = kind === 'reviewer' ? 'code review' : '安全审计';
let reviewRunId: string | null = null;
try {
const review = store.startRun(task.id, kind, { worktree: wt.dir, branch: wt.branch });
reviewRunId = review.id;
log.info(`${label} 任务 ${task.id} run=${review.id} model=${pickModel(task, project, 'reviewer')}`);
const rv = await fn(task, project, wt, review.id, executorReport);
store.finishRun(review.id, 'succeeded', {
transcriptRef: rv.transcriptRef ?? undefined,
claudeSessionId: rv.sessionId ?? undefined,
});
log.info(`任务 ${task.id} ${label} 完成 verdict=${rv.verdict ?? '(未解析到)'}`);
return { summary: rv.summary, verdict: rv.verdict };
} catch (e) {
const reMsg = (e as Error).message;
if (reviewRunId) {
try { store.finishRun(reviewRunId, 'failed', { error: reMsg }); } catch { /* 收尾失败不影响主流程 */ }
}
log.error(`任务 ${task.id} ${label} 失败(不挡任务,照常进 exec_review):${reMsg}`);
return { summary: `自动复审失败:${reMsg}`, verdict: null };
}
}
/** 单任务全流程:executing → worktree → run → verify → code review run → 安全审计 run(均失败不挡)→ setResult(四字段) → exec_review / failed(重试) */
async function executeTask(project: Project, task: Task): Promise<void> {
let runId: string | null = null;
let runClosed = false;
try {
store.transition(task.id, 'executing', { by: 'orchestrator' });
const wt = await d.createWorktree(project.repoPath, task.id, project.defaultBranch);
const run = store.startRun(task.id, 'executor', { worktree: wt.dir, branch: wt.branch });
runId = run.id;
log.info(`执行任务 ${task.id}${task.title}」 run=${run.id} worktree=${wt.dir}`);
const rr = await d.runner(task, project, wt, run.id);
if (!rr.ok) {
store.finishRun(run.id, 'failed', {
error: rr.error ?? '执行失败',
transcriptRef: rr.transcriptRef ?? undefined,
claudeSessionId: rr.sessionId ?? undefined,
});
runClosed = true;
throw new Error(rr.error ?? '执行失败');
}
const vr = await d.verify(project, wt.dir, run.id);
if (!vr.ok) {
store.finishRun(run.id, 'failed', {
error: vr.error ?? 'verify 失败',
transcriptRef: rr.transcriptRef ?? undefined,
claudeSessionId: rr.sessionId ?? undefined,
});
runClosed = true;
throw new Error(vr.error ?? 'verify 失败');
}
const diff = await d.worktreeDiff(project.repoPath, wt.dir, wt.branch, project.defaultBranch);
// 双复审(顺序):code review runkind=reviewer)→ 安全审计 runkind=security)。任一失败不挡任务。
const code = await runOneReview('reviewer', d.reviewCode, task, project, wt, rr.finalText ?? '');
const sec = await runOneReview('security', d.reviewSecurity, task, project, wt, rr.finalText ?? '');
store.setResult(task.id, {
branch: wt.branch, worktree: wt.dir,
diffSummary: diff.diffSummary, commits: diff.commits, prUrl: null,
summary: code.summary, verdict: code.verdict,
securitySummary: sec.summary, securityVerdict: sec.verdict,
});
store.transition(task.id, 'exec_review', { by: 'orchestrator', runId: run.id });
store.finishRun(run.id, 'succeeded', {
transcriptRef: rr.transcriptRef ?? undefined,
claudeSessionId: rr.sessionId ?? undefined,
});
log.info(`任务 ${task.id} 执行完成 → exec_review${diff.commits.length} commits`);
} catch (e) {
const msg = (e as Error).message;
log.error(`任务 ${task.id} 执行失败:${msg}`);
try {
if (runId && !runClosed) {
store.finishRun(runId, 'failed', { error: msg }); // worktree 创建后抛错(diff 等)时收尾
} else if (!runId) {
// run 还没建(如 createWorktree 失败):补记一条 failed run,保证重试计数不漏
const r = store.startRun(task.id, 'executor');
store.finishRun(r.id, 'failed', { error: msg });
}
store.transition(task.id, 'failed', { by: 'orchestrator', error: msg });
const allFailed = store.listRuns(task.id).filter((r) => r.kind === 'executor' && r.status === 'failed').length;
// 相对于上次手动重投基线的净失败次数(retryBaseline 在 requeueTask 时设置)
const netFailed = Math.max(0, allFailed - task.retryBaseline);
const priorNetFailed = Math.max(0, netFailed - 1); // 不含本次
const maxRetries = project.maxRetries;
if (priorNetFailed < maxRetries) {
const attempt = priorNetFailed + 1; // 本次是第几次重试
const backoffMs = computeBackoffMs(attempt);
backoffUntil.set(task.id, d.nowMs() + backoffMs);
store.transition(task.id, 'queued', { by: 'orchestrator', retry: attempt, backoffMs });
log.info(`任务 ${task.id} 重新入队(第 ${attempt} 次重试,退避 ${backoffMs}ms 后可领)`);
} else {
backoffUntil.delete(task.id);
store.transition(task.id, 'needs_attention', { by: 'orchestrator', allFailed, netFailed });
log.error(`任务 ${task.id} 净失败 ${netFailed} 次(上限 ${maxRetries})→ needs_attention`);
}
} catch (e2) {
log.error(`任务 ${task.id} 失败收尾出错:${(e2 as Error).message}`);
}
}
}
function tick(): void {
try {
for (const p of store.listProjects()) {
if (p.status !== 'active' || p.autonomy === 'manual') continue;
let active = 0;
for (const pid of inflight.values()) if (pid === p.id) active++;
if (active >= p.concurrency) continue;
for (const { task, score } of claimable(p)) {
if (active >= p.concurrency) break;
try {
if (task.status === 'ready') store.transition(task.id, 'queued', { by: 'orchestrator', score });
} catch (e) {
log.error(`任务 ${task.id} 入队失败:${(e as Error).message}`);
continue;
}
log.info(`领取任务 ${task.id}${task.title}」score=${score} model=${pickModel(task, p, 'executor')}`);
inflight.set(task.id, p.id);
active++;
const job: Promise<void> = executeTask(p, { ...task, status: 'queued' })
.catch((e) => log.error(`任务 ${task.id} 执行异常:${(e as Error).message}`))
.finally(() => {
inflight.delete(task.id);
pending.delete(job);
});
pending.add(job);
}
}
} catch (e) {
log.error(`编排器轮询失败:${(e as Error).message}`);
}
}
async function drain(): Promise<void> {
while (pending.size > 0) await Promise.allSettled([...pending]);
}
return { tick, drain, inflight };
}
/**
* 接线入口:MAESTRO_ORCH_INTERVAL(秒)控制轮询间隔,默认 15,0=关闭。
* 返回 timer 供 shutdown 时 clearInterval。
*/
export function startOrchestrator(
store: Store,
app: { log: OrchestratorLogger },
deps: Partial<OrchestratorDeps> = {},
): NodeJS.Timeout | null {
const intervalSec = Number(process.env.MAESTRO_ORCH_INTERVAL ?? 15);
if (!Number.isFinite(intervalSec) || intervalSec <= 0) {
app.log.info('编排器已关闭(MAESTRO_ORCH_INTERVAL=0');
return null;
}
const orch = createOrchestrator(store, app.log, deps);
const timer = setInterval(() => orch.tick(), intervalSec * 1000);
timer.unref();
app.log.info(`编排器已启用:每 ${intervalSec}s 一轮领取(autonomy≠manual 的 active 项目)`);
return timer;
}