feat: 重试与超时策略细化（可配 + 退避）tsk_N5wO3Armums2
## 变更摘要 ### 1. 项目级可配重试上限与超时 - `Project` 新增 `maxRetries`（默认 2）和 `timeoutMs`（默认 30min = 1800000ms）字段 - SQLite schema：`projects` 表补 `max_retries` / `timeout_ms` 列（带 DEFAULT 的轻量迁移） - `createProject` / `patchProject` 支持设置 / 校验新字段（maxRetries>=0, timeoutMs>=1000） - API `POST /api/projects` 与 `PATCH /api/projects/:id` 透传新字段 - `runner.ts` 将 `project.timeoutMs` 传给 `runClaude`，不再固定 30min ### 2. 失败重试指数退避 - 编排器内存维护 `backoffUntil` Map（taskId → nextRetryAt），daemon 重启后清空 - 退避公式：`min(30s * 2^(attempt-1), 10min)`，第 1 次 30s / 第 2 次 60s / ... - `claimable()` 过滤退避冷却中的任务 - `nowMs` 注入点（默认 `Date.now`）使测试可快进时钟验证退避行为 ### 3. needs_attention 一键重投 - `Task` 新增 `retryBaseline` 字段（默认 0）：记录上次重投时的失败 run 基线 - `Store.requeueTask(taskId)`：设置基线 = 当前失败 run 数 → 转 `queued`（仅限 needs_attention） - 编排器用净失败数（`allFailed - retryBaseline`）判断是否已耗尽重试次数 - API `POST /api/tasks/:id/requeue` + MCP `requeue_task` 工具 ### 4. 测试 - 重命名 `MAX_RETRIES` → `DEFAULT_MAX_RETRIES`，新增导出 `computeBackoffMs` - 新增测试（共 +18）：maxRetries=1/0、退避时序（精确 ms 边界）、重投后基线重置 - 迁移测试扩展：验证旧库补列后 maxRetries/timeoutMs/retryBaseline 使用默认值 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -26,6 +26,9 @@ export function openDb(file: string): Database.Database {
|
||||
ensureColumn(db, 'projects', 'last_sync_at', 'last_sync_at TEXT');
|
||||
ensureColumn(db, 'projects', 'logo', 'logo TEXT'); // 自定义 logo（URL/仓库内相对路径，null=自动解析）
|
||||
ensureColumn(db, 'projects', 'sort_order', 'sort_order INTEGER NOT NULL DEFAULT 0'); // 侧栏排序
|
||||
ensureColumn(db, 'projects', 'max_retries', 'max_retries INTEGER NOT NULL DEFAULT 2'); // 失败后最大自动重试次数
|
||||
ensureColumn(db, 'projects', 'timeout_ms', 'timeout_ms INTEGER NOT NULL DEFAULT 1800000'); // 单次执行超时毫秒
|
||||
ensureColumn(db, 'tasks', 'retry_baseline', 'retry_baseline INTEGER NOT NULL DEFAULT 0'); // 手动重投时的失败基线
|
||||
const schema = readFileSync(join(HERE, 'schema.sql'), 'utf8');
|
||||
db.exec(schema);
|
||||
return db;
|
||||
|
||||
@@ -7,14 +7,16 @@ import type { RunKind, RunStatus, EventType } from '../model/types.js';
|
||||
export interface ProjectRow {
|
||||
id: string; name: string; repo_path: string; default_branch: string;
|
||||
verify_cmd: string | null; autonomy: string; model: string | null;
|
||||
concurrency: number; status: string; created_at: string;
|
||||
concurrency: number; max_retries: number; timeout_ms: number;
|
||||
status: string; created_at: string;
|
||||
last_sync_at: string | null; logo: string | null; sort_order: number;
|
||||
}
|
||||
export interface TaskRow {
|
||||
id: string; project_id: string; parent_id: string | null; depth: number;
|
||||
title: string; complexity: string; status: string; priority: number;
|
||||
deps: string; plan: string | null; spec: string | null; operations: string | null;
|
||||
result: string | null; assignee: string | null; created_at: string; updated_at: string;
|
||||
result: string | null; assignee: string | null; retry_baseline: number;
|
||||
created_at: string; updated_at: string;
|
||||
source_ref: string | null;
|
||||
}
|
||||
export interface ApprovalRow {
|
||||
@@ -34,7 +36,8 @@ export function rowToProject(r: ProjectRow): Project {
|
||||
return {
|
||||
id: r.id, name: r.name, repoPath: r.repo_path, defaultBranch: r.default_branch,
|
||||
verifyCmd: r.verify_cmd, autonomy: r.autonomy as Autonomy, model: r.model,
|
||||
concurrency: r.concurrency, status: r.status as Project['status'], createdAt: r.created_at,
|
||||
concurrency: r.concurrency, maxRetries: r.max_retries ?? 2, timeoutMs: r.timeout_ms ?? 1_800_000,
|
||||
status: r.status as Project['status'], createdAt: r.created_at,
|
||||
lastSyncAt: r.last_sync_at, logo: r.logo, sortOrder: r.sort_order,
|
||||
};
|
||||
}
|
||||
@@ -63,7 +66,8 @@ export function rowToTask(r: TaskRow, approvals: ApprovalRecord[] = []): Task {
|
||||
priority: r.priority, deps: JSON.parse(r.deps) as string[],
|
||||
plan: r.plan, spec: r.spec, operations: r.operations,
|
||||
approvals, result: r.result ? parseResult(r.result) : null,
|
||||
assignee: r.assignee as Task['assignee'], createdAt: r.created_at, updatedAt: r.updated_at,
|
||||
assignee: r.assignee as Task['assignee'], retryBaseline: r.retry_baseline ?? 0,
|
||||
createdAt: r.created_at, updatedAt: r.updated_at,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ CREATE TABLE IF NOT EXISTS projects (
|
||||
autonomy TEXT NOT NULL DEFAULT 'manual', -- manual | auto-easy | auto-approved
|
||||
model TEXT,
|
||||
concurrency INTEGER NOT NULL DEFAULT 1,
|
||||
max_retries INTEGER NOT NULL DEFAULT 2, -- 失败后最大自动重试次数
|
||||
timeout_ms INTEGER NOT NULL DEFAULT 1800000, -- 单次执行超时毫秒（默认 30min）
|
||||
status TEXT NOT NULL DEFAULT 'active', -- active | paused
|
||||
created_at TEXT NOT NULL,
|
||||
last_sync_at TEXT, -- 最近一次 todo.json 同步时间
|
||||
@@ -31,8 +33,9 @@ CREATE TABLE IF NOT EXISTS tasks (
|
||||
plan TEXT, -- Hard：分析 + 拆解
|
||||
spec TEXT, -- Medium：改动 + 理由
|
||||
operations TEXT, -- Easy：将执行的操作
|
||||
result TEXT, -- JSON TaskResult
|
||||
assignee TEXT, -- agent | human
|
||||
result TEXT, -- JSON TaskResult
|
||||
assignee TEXT, -- agent | human
|
||||
retry_baseline INTEGER NOT NULL DEFAULT 0, -- 上次手动重投时已有的失败 run 数（重置重试计数用）
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
source_ref TEXT -- 旧 todo 来源标识（todo:17 / todo:17/1A），项目内唯一
|
||||
|
||||
+48
-6
@@ -20,6 +20,7 @@ export class StoreError extends Error {}
|
||||
export interface CreateProjectInput {
|
||||
name: string; repoPath: string; defaultBranch?: string; verifyCmd?: string | null;
|
||||
autonomy?: Autonomy; model?: string | null; concurrency?: number;
|
||||
maxRetries?: number; timeoutMs?: number;
|
||||
}
|
||||
export interface CreateTaskInput {
|
||||
projectId: string; title: string; complexity: Complexity;
|
||||
@@ -28,6 +29,7 @@ export interface CreateTaskInput {
|
||||
export interface PatchProjectInput {
|
||||
autonomy?: Autonomy; concurrency?: number; verifyCmd?: string | null;
|
||||
model?: string | null; status?: 'active' | 'paused'; logo?: string | null;
|
||||
maxRetries?: number; timeoutMs?: number;
|
||||
}
|
||||
export interface PatchTaskInput {
|
||||
title?: string; priority?: number; complexity?: Complexity;
|
||||
@@ -89,17 +91,25 @@ export class Store {
|
||||
|
||||
// ---------- Projects ----------
|
||||
createProject(input: CreateProjectInput): Project {
|
||||
if (input.maxRetries !== undefined && (!Number.isInteger(input.maxRetries) || input.maxRetries < 0)) {
|
||||
throw new StoreError('maxRetries 必须是 >=0 的整数');
|
||||
}
|
||||
if (input.timeoutMs !== undefined && (!Number.isFinite(input.timeoutMs) || input.timeoutMs < 1000)) {
|
||||
throw new StoreError('timeoutMs 必须是 >=1000 的数字（毫秒）');
|
||||
}
|
||||
const maxOrder = (this.db.prepare(`SELECT COALESCE(MAX(sort_order), -1) AS m FROM projects`).get() as { m: number }).m;
|
||||
const row: ProjectRow = {
|
||||
id: id('prj'), name: input.name, repo_path: input.repoPath,
|
||||
default_branch: input.defaultBranch ?? 'main', verify_cmd: input.verifyCmd ?? null,
|
||||
autonomy: input.autonomy ?? 'manual', model: input.model ?? null,
|
||||
concurrency: input.concurrency ?? 1, status: 'active', created_at: now(),
|
||||
concurrency: input.concurrency ?? 1, max_retries: input.maxRetries ?? 2,
|
||||
timeout_ms: input.timeoutMs ?? 1_800_000,
|
||||
status: 'active', created_at: now(),
|
||||
last_sync_at: null, logo: null, sort_order: maxOrder + 1,
|
||||
};
|
||||
this.db.prepare(
|
||||
`INSERT INTO projects (id,name,repo_path,default_branch,verify_cmd,autonomy,model,concurrency,status,created_at,last_sync_at,logo,sort_order)
|
||||
VALUES (@id,@name,@repo_path,@default_branch,@verify_cmd,@autonomy,@model,@concurrency,@status,@created_at,@last_sync_at,@logo,@sort_order)`,
|
||||
`INSERT INTO projects (id,name,repo_path,default_branch,verify_cmd,autonomy,model,concurrency,max_retries,timeout_ms,status,created_at,last_sync_at,logo,sort_order)
|
||||
VALUES (@id,@name,@repo_path,@default_branch,@verify_cmd,@autonomy,@model,@concurrency,@max_retries,@timeout_ms,@status,@created_at,@last_sync_at,@logo,@sort_order)`,
|
||||
).run(row);
|
||||
this.emit(row.id, null, 'task.created', { kind: 'project', name: row.name });
|
||||
return rowToProject(row);
|
||||
@@ -159,6 +169,18 @@ export class Store {
|
||||
if (patch.verifyCmd !== undefined) { sets.push('verify_cmd = @verify_cmd'); args.verify_cmd = patch.verifyCmd; }
|
||||
if (patch.model !== undefined) { sets.push('model = @model'); args.model = patch.model; }
|
||||
if (patch.logo !== undefined) { sets.push('logo = @logo'); args.logo = patch.logo; }
|
||||
if (patch.maxRetries !== undefined) {
|
||||
if (!Number.isInteger(patch.maxRetries) || patch.maxRetries < 0) {
|
||||
throw new StoreError('maxRetries 必须是 >=0 的整数');
|
||||
}
|
||||
sets.push('max_retries = @max_retries'); args.max_retries = patch.maxRetries;
|
||||
}
|
||||
if (patch.timeoutMs !== undefined) {
|
||||
if (!Number.isFinite(patch.timeoutMs) || patch.timeoutMs < 1000) {
|
||||
throw new StoreError('timeoutMs 必须是 >=1000 的数字（毫秒）');
|
||||
}
|
||||
sets.push('timeout_ms = @timeout_ms'); args.timeout_ms = patch.timeoutMs;
|
||||
}
|
||||
|
||||
if (sets.length > 0) {
|
||||
this.db.prepare(`UPDATE projects SET ${sets.join(', ')} WHERE id = @id`).run(args);
|
||||
@@ -209,12 +231,12 @@ export class Store {
|
||||
id: id('tsk'), project_id: input.projectId, parent_id: input.parentId ?? null, depth,
|
||||
title: input.title, complexity: input.complexity, status, priority: input.priority ?? 1,
|
||||
deps: JSON.stringify(input.deps ?? []), plan: null, spec: null, operations: null,
|
||||
result: null, assignee: null, created_at: now(), updated_at: now(),
|
||||
result: null, assignee: null, retry_baseline: 0, created_at: now(), updated_at: now(),
|
||||
source_ref: null,
|
||||
};
|
||||
this.db.prepare(
|
||||
`INSERT INTO tasks (id,project_id,parent_id,depth,title,complexity,status,priority,deps,plan,spec,operations,result,assignee,created_at,updated_at,source_ref)
|
||||
VALUES (@id,@project_id,@parent_id,@depth,@title,@complexity,@status,@priority,@deps,@plan,@spec,@operations,@result,@assignee,@created_at,@updated_at,@source_ref)`,
|
||||
`INSERT INTO tasks (id,project_id,parent_id,depth,title,complexity,status,priority,deps,plan,spec,operations,result,assignee,retry_baseline,created_at,updated_at,source_ref)
|
||||
VALUES (@id,@project_id,@parent_id,@depth,@title,@complexity,@status,@priority,@deps,@plan,@spec,@operations,@result,@assignee,@retry_baseline,@created_at,@updated_at,@source_ref)`,
|
||||
).run(row);
|
||||
this.emit(input.projectId, row.id, 'task.created', { title: row.title, complexity: row.complexity, status });
|
||||
return rowToTask(row);
|
||||
@@ -509,6 +531,26 @@ export class Store {
|
||||
return this.getTask(taskId)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动一键重投 needs_attention 任务：
|
||||
* 1. 记录当前失败 run 数为新基线（重置重试计数）；
|
||||
* 2. 转 needs_attention → queued，让编排器下一轮重新领取。
|
||||
*/
|
||||
requeueTask(taskId: string): Task {
|
||||
const row = this.getTaskRow(taskId);
|
||||
if (!row) throw new StoreError(`任务不存在: ${taskId}`);
|
||||
if (row.status !== 'needs_attention') {
|
||||
throw new StoreError(`只有 needs_attention 状态的任务才可重投（当前状态：${row.status}）`);
|
||||
}
|
||||
const failedCount = (this.db.prepare(
|
||||
`SELECT COUNT(*) AS n FROM runs WHERE task_id = ? AND kind = 'executor' AND status = 'failed'`,
|
||||
).get(taskId) as { n: number }).n;
|
||||
this.db.prepare(`UPDATE tasks SET retry_baseline = ?, updated_at = ? WHERE id = ?`).run(failedCount, now(), taskId);
|
||||
this.transition(taskId, 'queued', { by: 'user', action: 'requeue' });
|
||||
this.emit(row.project_id, taskId, 'task.updated', { field: 'retryBaseline', retryBaseline: failedCount });
|
||||
return this.getTask(taskId)!;
|
||||
}
|
||||
|
||||
// ---------- Runs ----------
|
||||
startRun(taskId: string, kind: Run['kind'], fields: Partial<Pick<Run, 'worktree' | 'branch'>> = {}): Run {
|
||||
const row = this.getTaskRow(taskId);
|
||||
|
||||
Reference in New Issue
Block a user