wangjia/maestro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 重试与超时策略细化（可配 + 退避）tsk_N5wO3Armums2

Browse Source 
## 变更摘要

### 1. 项目级可配重试上限与超时
- `Project` 新增 `maxRetries`（默认 2）和 `timeoutMs`（默认 30min = 1800000ms）字段
- SQLite schema：`projects` 表补 `max_retries` / `timeout_ms` 列（带 DEFAULT 的轻量迁移）
- `createProject` / `patchProject` 支持设置 / 校验新字段（maxRetries>=0, timeoutMs>=1000）
- API `POST /api/projects` 与 `PATCH /api/projects/:id` 透传新字段
- `runner.ts` 将 `project.timeoutMs` 传给 `runClaude`，不再固定 30min

### 2. 失败重试指数退避
- 编排器内存维护 `backoffUntil` Map（taskId → nextRetryAt），daemon 重启后清空
- 退避公式：`min(30s * 2^(attempt-1), 10min)`，第 1 次 30s / 第 2 次 60s / ...
- `claimable()` 过滤退避冷却中的任务
- `nowMs` 注入点（默认 `Date.now`）使测试可快进时钟验证退避行为

### 3. needs_attention 一键重投
- `Task` 新增 `retryBaseline` 字段（默认 0）：记录上次重投时的失败 run 基线
- `Store.requeueTask(taskId)`：设置基线 = 当前失败 run 数 → 转 `queued`（仅限 needs_attention）
- 编排器用净失败数（`allFailed - retryBaseline`）判断是否已耗尽重试次数
- API `POST /api/tasks/:id/requeue` + MCP `requeue_task` 工具

### 4. 测试
- 重命名 `MAX_RETRIES` → `DEFAULT_MAX_RETRIES`，新增导出 `computeBackoffMs`
- 新增测试（共 +18）：maxRetries=1/0、退避时序（精确 ms 边界）、重投后基线重置
- 迁移测试扩展：验证旧库补列后 maxRetries/timeoutMs/retryBaseline 使用默认值

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-13 04:04:41 +08:00
parent bb6186902a
commit c89d6d129c
12 changed files with 347 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/daemon/orchestrator.ts
+33 -10
View File
@@ -7,8 +7,17 @@ import { runVerify, type VerifyFn } from '../executor/verify.js';
import { reviewCode, reviewSecurity, type ReviewerFn } from '../executor/reviewer.js';
import { pickModel } from '../executor/models.js';
/** 失败后最多自动重试次数（重试 2 次 = 最多 3 次执行），之后 → needs_attention */
export const MAX_RETRIES = 2;
/** 失败后默认最多自动重试次数（项目级 maxRetries 未设置时回退此值） */
export const DEFAULT_MAX_RETRIES = 2;
/** 指数退避基础等待（第 n 次重试等待 BASE * 2^(n-1)，上限 MAX） */
const BACKOFF_BASE_MS = 30_000; // 30s
const BACKOFF_MAX_MS = 10 * 60_000; // 10min
/** 第 attempt 次重试（1-indexed）的等待毫秒 */
export function computeBackoffMs(attempt: number): number {
return Math.min(BACKOFF_BASE_MS * (2 ** (attempt - 1)), BACKOFF_MAX_MS);
}
export interface OrchestratorLogger {
info(msg: string): void;
@@ -23,6 +32,8 @@ export interface OrchestratorDeps {
verify: VerifyFn;
createWorktree: (repoPath: string, taskId: string, baseBranch: string) => Promise<WorktreeInfo>;
worktreeDiff: (repoPath: string, dir: string, branch: string, baseBranch: string) => Promise<WorktreeDiff>;
/** 当前时间戳（ms）；测试注入可控时钟（默认 Date.now） */
nowMs: () => number;
}
export interface Orchestrator {
@@ -41,15 +52,18 @@ export interface Orchestrator {
* 成功 → setResult + exec_review；失败 → failed → 重试 ≤MAX_RETRIES 次 → needs_attention。
*/
export function createOrchestrator(store: Store, log: OrchestratorLogger, deps: Partial<OrchestratorDeps> = {}): Orchestrator {
const d: OrchestratorDeps = { runner: runTask, reviewCode, reviewSecurity, verify: runVerify, createWorktree, worktreeDiff, ...deps };
const d: OrchestratorDeps = { runner: runTask, reviewCode, reviewSecurity, verify: runVerify, createWorktree, worktreeDiff, nowMs: Date.now, ...deps };
const inflight = new Map<string, string>(); // taskId → projectId
const pending = new Set<Promise<void>>();
/** 退避表：taskId → 下次可领取时间戳（ms）；daemon 重启后清空，已有 queued 任务即刻可领 */
const backoffUntil = new Map<string, number>();
/**
* 本项目可领取的任务：queued（重试/孤儿）+ ready 叶子且 deps 全 done；auto-easy 只挑 easy。
* 按调度分降序返回（score = 自身分 + 已完成依赖分 + 等待解锁的 blocked 任务分，见 model/scoring.ts）。
*/
function claimable(project: Project): Array<{ task: Task; score: number }> {
const nowMs = d.nowMs();
const tasks = store.listTasks(project.id);
const byId = new Map(tasks.map((t) => [t.id, t]));
const parents = new Set(tasks.filter((t) => t.parentId).map((t) => t.parentId as string));
@@ -59,6 +73,8 @@ export function createOrchestrator(store: Store, log: OrchestratorLogger, deps:
if (t.status !== 'ready' && t.status !== 'queued') return false;
if (parents.has(t.id)) return false; // 非叶子（容器）跳过
if (easyOnly && t.complexity !== 'easy') return false;
const until = backoffUntil.get(t.id);
if (until !== undefined && nowMs < until) return false; // 退避冷却中
return t.deps.every((dep) => byId.get(dep)?.status === 'done');
});
return rankByScore(candidates, tasks);
@@ -159,14 +175,21 @@ export function createOrchestrator(store: Store, log: OrchestratorLogger, deps:
store.finishRun(r.id, 'failed', { error: msg });
}
store.transition(task.id, 'failed', { by: 'orchestrator', error: msg });
const failedRuns = store.listRuns(task.id).filter((r) => r.kind === 'executor' && r.status === 'failed').length;
const priorFailed = Math.max(0, failedRuns - 1); // 不含本次
if (priorFailed < MAX_RETRIES) {
store.transition(task.id, 'queued', { by: 'orchestrator', retry: priorFailed + 1 });
log.info(`任务 ${task.id} 重新入队（第 ${priorFailed + 1} 次重试，下一轮领取）`);
const allFailed = store.listRuns(task.id).filter((r) => r.kind === 'executor' && r.status === 'failed').length;
// 相对于上次手动重投基线的净失败次数（retryBaseline 在 requeueTask 时设置）
const netFailed = Math.max(0, allFailed - task.retryBaseline);
const priorNetFailed = Math.max(0, netFailed - 1); // 不含本次
const maxRetries = project.maxRetries;
if (priorNetFailed < maxRetries) {
const attempt = priorNetFailed + 1; // 本次是第几次重试
const backoffMs = computeBackoffMs(attempt);
backoffUntil.set(task.id, d.nowMs() + backoffMs);
store.transition(task.id, 'queued', { by: 'orchestrator', retry: attempt, backoffMs });
log.info(`任务 ${task.id} 重新入队（第 ${attempt} 次重试，退避 ${backoffMs}ms 后可领）`);
} else {
store.transition(task.id, 'needs_attention', { by: 'orchestrator', failedRuns });
log.error(`任务 ${task.id} 连续失败 ${failedRuns} 次 → needs_attention`);
backoffUntil.delete(task.id);
store.transition(task.id, 'needs_attention', { by: 'orchestrator', allFailed, netFailed });
log.error(`任务 ${task.id} 净失败 ${netFailed} 次（上限 ${maxRetries}）→ needs_attention`);
}
} catch (e2) {
log.error(`任务 ${task.id} 失败收尾出错：${(e2 as Error).message}`);