wangjia/maestro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 重试与超时策略细化（可配 + 退避）tsk_N5wO3Armums2

Browse Source 
## 变更摘要

### 1. 项目级可配重试上限与超时
- `Project` 新增 `maxRetries`（默认 2）和 `timeoutMs`（默认 30min = 1800000ms）字段
- SQLite schema：`projects` 表补 `max_retries` / `timeout_ms` 列（带 DEFAULT 的轻量迁移）
- `createProject` / `patchProject` 支持设置 / 校验新字段（maxRetries>=0, timeoutMs>=1000）
- API `POST /api/projects` 与 `PATCH /api/projects/:id` 透传新字段
- `runner.ts` 将 `project.timeoutMs` 传给 `runClaude`，不再固定 30min

### 2. 失败重试指数退避
- 编排器内存维护 `backoffUntil` Map（taskId → nextRetryAt），daemon 重启后清空
- 退避公式：`min(30s * 2^(attempt-1), 10min)`，第 1 次 30s / 第 2 次 60s / ...
- `claimable()` 过滤退避冷却中的任务
- `nowMs` 注入点（默认 `Date.now`）使测试可快进时钟验证退避行为

### 3. needs_attention 一键重投
- `Task` 新增 `retryBaseline` 字段（默认 0）：记录上次重投时的失败 run 基线
- `Store.requeueTask(taskId)`：设置基线 = 当前失败 run 数 → 转 `queued`（仅限 needs_attention）
- 编排器用净失败数（`allFailed - retryBaseline`）判断是否已耗尽重试次数
- API `POST /api/tasks/:id/requeue` + MCP `requeue_task` 工具

### 4. 测试
- 重命名 `MAX_RETRIES` → `DEFAULT_MAX_RETRIES`，新增导出 `computeBackoffMs`
- 新增测试（共 +18）：maxRetries=1/0、退避时序（精确 ms 边界）、重投后基线重置
- 迁移测试扩展：验证旧库补列后 maxRetries/timeoutMs/retryBaseline 使用默认值

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-13 04:04:41 +08:00
parent bb6186902a
commit c89d6d129c
12 changed files with 347 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/store/store.ts
+48 -6
View File
@@ -20,6 +20,7 @@ export class StoreError extends Error {}
export interface CreateProjectInput {
name: string; repoPath: string; defaultBranch?: string; verifyCmd?: string | null;
autonomy?: Autonomy; model?: string | null; concurrency?: number;
maxRetries?: number; timeoutMs?: number;
}
export interface CreateTaskInput {
projectId: string; title: string; complexity: Complexity;
@@ -28,6 +29,7 @@ export interface CreateTaskInput {
export interface PatchProjectInput {
autonomy?: Autonomy; concurrency?: number; verifyCmd?: string | null;
model?: string | null; status?: 'active' | 'paused'; logo?: string | null;
maxRetries?: number; timeoutMs?: number;
}
export interface PatchTaskInput {
title?: string; priority?: number; complexity?: Complexity;
@@ -89,17 +91,25 @@ export class Store {
// ---------- Projects ----------
createProject(input: CreateProjectInput): Project {
if (input.maxRetries !== undefined && (!Number.isInteger(input.maxRetries) || input.maxRetries < 0)) {
throw new StoreError('maxRetries 必须是 >=0 的整数');
}
if (input.timeoutMs !== undefined && (!Number.isFinite(input.timeoutMs) || input.timeoutMs < 1000)) {
throw new StoreError('timeoutMs 必须是 >=1000 的数字（毫秒）');
}
const maxOrder = (this.db.prepare(`SELECT COALESCE(MAX(sort_order), -1) AS m FROM projects`).get() as { m: number }).m;
const row: ProjectRow = {
id: id('prj'), name: input.name, repo_path: input.repoPath,
default_branch: input.defaultBranch ?? 'main', verify_cmd: input.verifyCmd ?? null,
autonomy: input.autonomy ?? 'manual', model: input.model ?? null,
concurrency: input.concurrency ?? 1, status: 'active', created_at: now(),
concurrency: input.concurrency ?? 1, max_retries: input.maxRetries ?? 2,
timeout_ms: input.timeoutMs ?? 1_800_000,
status: 'active', created_at: now(),
last_sync_at: null, logo: null, sort_order: maxOrder + 1,
};
this.db.prepare(
`INSERT INTO projects (id,name,repo_path,default_branch,verify_cmd,autonomy,model,concurrency,status,created_at,last_sync_at,logo,sort_order)
VALUES (@id,@name,@repo_path,@default_branch,@verify_cmd,@autonomy,@model,@concurrency,@status,@created_at,@last_sync_at,@logo,@sort_order)`,
`INSERT INTO projects (id,name,repo_path,default_branch,verify_cmd,autonomy,model,concurrency,max_retries,timeout_ms,status,created_at,last_sync_at,logo,sort_order)
VALUES (@id,@name,@repo_path,@default_branch,@verify_cmd,@autonomy,@model,@concurrency,@max_retries,@timeout_ms,@status,@created_at,@last_sync_at,@logo,@sort_order)`,
).run(row);
this.emit(row.id, null, 'task.created', { kind: 'project', name: row.name });
return rowToProject(row);
@@ -159,6 +169,18 @@ export class Store {
if (patch.verifyCmd !== undefined) { sets.push('verify_cmd = @verify_cmd'); args.verify_cmd = patch.verifyCmd; }
if (patch.model !== undefined) { sets.push('model = @model'); args.model = patch.model; }
if (patch.logo !== undefined) { sets.push('logo = @logo'); args.logo = patch.logo; }
if (patch.maxRetries !== undefined) {
if (!Number.isInteger(patch.maxRetries) || patch.maxRetries < 0) {
throw new StoreError('maxRetries 必须是 >=0 的整数');
}
sets.push('max_retries = @max_retries'); args.max_retries = patch.maxRetries;
}
if (patch.timeoutMs !== undefined) {
if (!Number.isFinite(patch.timeoutMs) || patch.timeoutMs < 1000) {
throw new StoreError('timeoutMs 必须是 >=1000 的数字（毫秒）');
}
sets.push('timeout_ms = @timeout_ms'); args.timeout_ms = patch.timeoutMs;
}
if (sets.length > 0) {
this.db.prepare(`UPDATE projects SET ${sets.join(', ')} WHERE id = @id`).run(args);
@@ -209,12 +231,12 @@ export class Store {
id: id('tsk'), project_id: input.projectId, parent_id: input.parentId ?? null, depth,
title: input.title, complexity: input.complexity, status, priority: input.priority ?? 1,
deps: JSON.stringify(input.deps ?? []), plan: null, spec: null, operations: null,
result: null, assignee: null, created_at: now(), updated_at: now(),
result: null, assignee: null, retry_baseline: 0, created_at: now(), updated_at: now(),
source_ref: null,
};
this.db.prepare(
`INSERT INTO tasks (id,project_id,parent_id,depth,title,complexity,status,priority,deps,plan,spec,operations,result,assignee,created_at,updated_at,source_ref)
VALUES (@id,@project_id,@parent_id,@depth,@title,@complexity,@status,@priority,@deps,@plan,@spec,@operations,@result,@assignee,@created_at,@updated_at,@source_ref)`,
`INSERT INTO tasks (id,project_id,parent_id,depth,title,complexity,status,priority,deps,plan,spec,operations,result,assignee,retry_baseline,created_at,updated_at,source_ref)
VALUES (@id,@project_id,@parent_id,@depth,@title,@complexity,@status,@priority,@deps,@plan,@spec,@operations,@result,@assignee,@retry_baseline,@created_at,@updated_at,@source_ref)`,
).run(row);
this.emit(input.projectId, row.id, 'task.created', { title: row.title, complexity: row.complexity, status });
return rowToTask(row);
@@ -509,6 +531,26 @@ export class Store {
return this.getTask(taskId)!;
}
/**
* 手动一键重投 needs_attention 任务：
* 1. 记录当前失败 run 数为新基线（重置重试计数）；
* 2. 转 needs_attention → queued，让编排器下一轮重新领取。
*/
requeueTask(taskId: string): Task {
const row = this.getTaskRow(taskId);
if (!row) throw new StoreError(`任务不存在: ${taskId}`);
if (row.status !== 'needs_attention') {
throw new StoreError(`只有 needs_attention 状态的任务才可重投（当前状态：${row.status}）`);
}
const failedCount = (this.db.prepare(
`SELECT COUNT(*) AS n FROM runs WHERE task_id = ? AND kind = 'executor' AND status = 'failed'`,
).get(taskId) as { n: number }).n;
this.db.prepare(`UPDATE tasks SET retry_baseline = ?, updated_at = ? WHERE id = ?`).run(failedCount, now(), taskId);
this.transition(taskId, 'queued', { by: 'user', action: 'requeue' });
this.emit(row.project_id, taskId, 'task.updated', { field: 'retryBaseline', retryBaseline: failedCount });
return this.getTask(taskId)!;
}
// ---------- Runs ----------
startRun(taskId: string, kind: Run['kind'], fields: Partial<Pick<Run, 'worktree' | 'branch'>> = {}): Run {
const row = this.getTaskRow(taskId);