feat: 重试与超时策略细化（可配 + 退避）tsk_N5wO3Armums2

## 变更摘要 ### 1. 项目级可配重试上限与超时 - `Project` 新增 `maxRetries`（默认 2）和 `timeoutMs`（默认 30min = 1800000ms）字段 - SQLite schema：`projects` 表补 `max_retries` / `timeout_ms` 列（带 DEFAULT 的轻量迁移） - `createProject` / `patchProject` 支持设置 / 校验新字段（maxRetries>=0, timeoutMs>=1000） - API `POST /api/projects` 与 `PATCH /api/projects/:id` 透传新字段 - `runner.ts` 将 `project.timeoutMs` 传给 `runClaude`，不再固定 30min ### 2. 失败重试指数退避 - 编排器内存维护 `backoffUntil` Map（taskId → nextRetryAt），daemon 重启后清空 - 退避公式：`min(30s * 2^(attempt-1), 10min)`，第 1 次 30s / 第 2 次 60s / ... - `claimable()` 过滤退避冷却中的任务 - `nowMs` 注入点（默认 `Date.now`）使测试可快进时钟验证退避行为 ### 3. needs_attention 一键重投 - `Task` 新增 `retryBaseline` 字段（默认 0）：记录上次重投时的失败 run 基线 - `Store.requeueTask(taskId)`：设置基线 = 当前失败 run 数 → 转 `queued`（仅限 needs_attention） - 编排器用净失败数（`allFailed - retryBaseline`）判断是否已耗尽重试次数 - API `POST /api/tasks/:id/requeue` + MCP `requeue_task` 工具 ### 4. 测试 - 重命名 `MAX_RETRIES` → `DEFAULT_MAX_RETRIES`，新增导出 `computeBackoffMs` - 新增测试（共 +18）：maxRetries=1/0、退避时序（精确 ms 边界）、重投后基线重置 - 迁移测试扩展：验证旧库补列后 maxRetries/timeoutMs/retryBaseline 使用默认值 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-13 04:04:41 +08:00
parent bb6186902a
commit c89d6d129c
12 changed files with 347 additions and 31 deletions
@@ -1,7 +1,7 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
 import { Store } from '../src/store/index.js';
-import { createOrchestrator, MAX_RETRIES, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
+import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
 import type { RunnerResult } from '../src/executor/runner.js';
 import type { ReviewResult } from '../src/executor/reviewer.js';
 import type { Autonomy } from '../src/model/types.js';
@@ -39,10 +39,17 @@ function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
    runner: async () => okRun,
    reviewCode: async () => okReview,
    reviewSecurity: async () => okSecurity,
+    nowMs: Date.now,
    ...overrides,
  };
 }

+/** 可前进的测试时钟（用于退避相关测试） */
+function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
+  let t = initialMs;
+  return { nowMs: () => t, advance: (ms) => { t += ms; } };
+}
+
 function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
  const store = new Store(':memory:');
  const p = store.createProject({
@@ -290,32 +297,165 @@ test('verify 不过：按失败处理（run failed + 重新入队）', async ()
  store.close();
 });

-test(`失败重试：重试 ${MAX_RETRIES} 次后 → needs_attention（共 ${MAX_RETRIES + 1} 次失败 run）`, async () => {
+test(`失败重试：重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention（共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run）`, async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });

  let attempts = 0;
+  // 使用快进时钟跳过退避冷却（每次 tick 前将时钟推进 10min）
+  const clock = makeClock();
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
+    nowMs: clock.nowMs,
  }));

-  for (let i = 1; i <= MAX_RETRIES; i++) {
+  for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
+    clock.advance(10 * 60_000);            // 跳过退避冷却
    orch.tick();
    await orch.drain();
    assert.equal(store.getTask(t.id)!.status, 'queued', `第 ${i} 次失败后应重新入队`);
  }

+  clock.advance(10 * 60_000);             // 跳过最后一次退避
  orch.tick();                              // 最后一次重试也失败
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
-  assert.equal(attempts, MAX_RETRIES + 1);
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);

  const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
-  assert.equal(failed.length, MAX_RETRIES + 1);
+  assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);

+  clock.advance(10 * 60_000);
  orch.tick();                              // needs_attention 不会再被领取
  await orch.drain();
-  assert.equal(attempts, MAX_RETRIES + 1);
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
+  store.close();
+});
+
+test('项目级 maxRetries=1：只重试 1 次就 → needs_attention', async () => {
+  const store = new Store(':memory:');
+  const p = store.createProject({
+    name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
+    autonomy: 'auto-easy', maxRetries: 1,
+  });
+  const projectId = p.id;
+  const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });
+
+  let attempts = 0;
+  const clock = makeClock();
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 第 1 次执行失败 → 重新入队（还有 1 次重试机会）
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+
+  // 第 2 次失败 → 超过 maxRetries=1 → needs_attention（快进时钟跳过退避）
+  clock.advance(10 * 60_000);
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, 2);
+  store.close();
+});
+
+test('项目级 maxRetries=0：首次失败直接 needs_attention（不重试）', async () => {
+  const store = new Store(':memory:');
+  const p = store.createProject({
+    name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
+    autonomy: 'auto-easy', maxRetries: 0,
+  });
+  const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });
+
+  let attempts = 0;
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+  }));
+
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, 1);
+  store.close();
+});
+
+test('computeBackoffMs：指数退避公式，30s 基数，上限 10min', () => {
+  assert.equal(computeBackoffMs(1), 30_000);
+  assert.equal(computeBackoffMs(2), 60_000);
+  assert.equal(computeBackoffMs(3), 120_000);
+  assert.equal(computeBackoffMs(4), 240_000);
+  assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
+  assert.equal(computeBackoffMs(100), 10 * 60_000);
+});
+
+test('退避冷却：首次失败重入队后，退避期内不被领取；退避过期后正常领取', async () => {
+  const { store, projectId } = setup('auto-easy');
+  const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
+
+  let attempts = 0;
+  // 使用可控时钟：初始时间 0，退避 30s（attempt=1 → BACKOFF_BASE=30000ms）
+  const clock = makeClock(0);
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 第 1 次失败（t=0）→ 重入队 + 退避（backoffUntil = 0 + 30000 = 30000ms）
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+  assert.equal(attempts, 1);
+
+  // t=29999：退避期内，不被领取
+  clock.advance(29_999);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, 1, '退避期内不应再次执行');
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+
+  // t=30001：退避过期，任务应被重新领取
+  clock.advance(2);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, 2, '退避过期后应再次执行');
+  store.close();
+});
+
+test('requeueTask：needs_attention → queued，重置重试基线，再次允许重试', async () => {
+  const { store, projectId } = setup('auto-easy');
+  const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });
+
+  let attempts = 0;
+  const clock = makeClock();
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 耗尽默认重试次数 → needs_attention（每次需快进时钟跳过退避）
+  for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
+    clock.advance(10 * 60_000);
+    orch.tick();
+    await orch.drain();
+  }
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
+
+  // 手动重投：重置基线，转 queued
+  const requeued = store.requeueTask(t.id);
+  assert.equal(requeued.status, 'queued');
+  assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);
+
+  // 重投后编排器应能再次执行（快进时钟确保无退避阻拦）
+  clock.advance(10 * 60_000);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
+  // 第 1 次净失败后应重入队（基线已重置，净失败=1 < maxRetries=2）
+  assert.equal(store.getTask(t.id)!.status, 'queued');
  store.close();
 });

@@ -208,3 +208,69 @@ test('容器收口：已拆解 Hard 的子任务全 done → 容器自动 done
  assert.equal(s.getTask(root.id).status, 'done');         // 子全 done → 容器自动 done
  s.close();
 });
+
+test('patchProject：maxRetries/timeoutMs 可配 + 校验', () => {
+  const s = freshStore();
+  const p = s.createProject({ name: 'cfg', repoPath: '/tmp/cfg-' + Math.random() });
+  // 默认值
+  assert.equal(p.maxRetries, 2);
+  assert.equal(p.timeoutMs, 1_800_000);
+
+  // 更新 maxRetries + timeoutMs
+  const updated = s.patchProject(p.id, { maxRetries: 5, timeoutMs: 60_000 });
+  assert.equal(updated.maxRetries, 5);
+  assert.equal(updated.timeoutMs, 60_000);
+
+  // 校验：maxRetries 必须 >=0 的整数
+  assert.throws(() => s.patchProject(p.id, { maxRetries: -1 }), /maxRetries/);
+  assert.throws(() => s.patchProject(p.id, { maxRetries: 1.5 }), /maxRetries/);
+
+  // 校验：timeoutMs 必须 >=1000
+  assert.throws(() => s.patchProject(p.id, { timeoutMs: 500 }), /timeoutMs/);
+  s.close();
+});
+
+test('createProject：maxRetries/timeoutMs 自定义初值', () => {
+  const s = freshStore();
+  const p = s.createProject({ name: 'custom', repoPath: '/tmp/custom-' + Math.random(), maxRetries: 0, timeoutMs: 120_000 });
+  assert.equal(p.maxRetries, 0);
+  assert.equal(p.timeoutMs, 120_000);
+  s.close();
+});
+
+test('requeueTask：needs_attention → queued，重置 retryBaseline', () => {
+  const s = freshStore();
+  const p = s.createProject({ name: 'rq', repoPath: '/tmp/rq-' + Math.random() });
+  const t = s.createTask({ projectId: p.id, title: 'flaky', complexity: 'easy' });
+
+  // 推进到 needs_attention（模拟 3 次失败 run）
+  s.transition(t.id, 'queued');
+  s.transition(t.id, 'executing');
+  s.transition(t.id, 'failed');
+  s.transition(t.id, 'queued');
+  s.transition(t.id, 'executing');
+  s.transition(t.id, 'failed');
+  s.transition(t.id, 'queued');
+  s.transition(t.id, 'executing');
+  s.transition(t.id, 'failed');
+  s.transition(t.id, 'needs_attention');
+
+  // 模拟 3 条 failed executor run
+  const r1 = s.startRun(t.id, 'executor');
+  s.finishRun(r1.id, 'failed', { error: 'boom1' });
+  const r2 = s.startRun(t.id, 'executor');
+  s.finishRun(r2.id, 'failed', { error: 'boom2' });
+  const r3 = s.startRun(t.id, 'executor');
+  s.finishRun(r3.id, 'failed', { error: 'boom3' });
+
+  assert.equal(s.getTask(t.id)!.retryBaseline, 0);
+
+  // 一键重投
+  const requeued = s.requeueTask(t.id);
+  assert.equal(requeued.status, 'queued');
+  assert.equal(requeued.retryBaseline, 3);   // 基线设为当前失败 run 数
+
+  // 只有 needs_attention 状态才可重投
+  assert.throws(() => s.requeueTask(t.id), /needs_attention/);
+  s.close();
+});
@@ -169,17 +169,23 @@ test('迁移：旧版库（无 source_ref/last_sync_at）打开后补列且数
  const projCols = (db.prepare(`PRAGMA table_info(projects)`).all() as Array<{ name: string }>).map((c) => c.name);
  const taskCols = (db.prepare(`PRAGMA table_info(tasks)`).all() as Array<{ name: string }>).map((c) => c.name);
  assert.ok(projCols.includes('last_sync_at'));
+  assert.ok(projCols.includes('max_retries'));
+  assert.ok(projCols.includes('timeout_ms'));
  assert.ok(taskCols.includes('source_ref'));
+  assert.ok(taskCols.includes('retry_baseline'));
  db.close();

-  // Store 能正常读旧数据，新字段为 null
+  // Store 能正常读旧数据，新字段使用默认值
  const s = new Store(file);
  const projects = s.listProjects();
  assert.equal(projects.length, 1);
  assert.equal(projects[0].name, '旧项目');
  assert.equal(projects[0].lastSyncAt, null);
+  assert.equal(projects[0].maxRetries, 2);         // 默认值
+  assert.equal(projects[0].timeoutMs, 1_800_000);  // 默认 30min
  const t = s.getTask('tsk_old');
  assert.equal(t?.title, '旧任务');
+  assert.equal(t?.retryBaseline, 0);               // 默认基线
  // 新方法在迁移后的旧库上可用
  s.setSourceRef('tsk_old', 'todo:99');
  assert.equal(s.getTaskBySourceRef('prj_old', 'todo:99')?.id, 'tsk_old');