feat: 重试与超时策略细化（可配 + 退避）tsk_N5wO3Armums2

## 变更摘要 ### 1. 项目级可配重试上限与超时 - `Project` 新增 `maxRetries`（默认 2）和 `timeoutMs`（默认 30min = 1800000ms）字段 - SQLite schema：`projects` 表补 `max_retries` / `timeout_ms` 列（带 DEFAULT 的轻量迁移） - `createProject` / `patchProject` 支持设置 / 校验新字段（maxRetries>=0, timeoutMs>=1000） - API `POST /api/projects` 与 `PATCH /api/projects/:id` 透传新字段 - `runner.ts` 将 `project.timeoutMs` 传给 `runClaude`，不再固定 30min ### 2. 失败重试指数退避 - 编排器内存维护 `backoffUntil` Map（taskId → nextRetryAt），daemon 重启后清空 - 退避公式：`min(30s * 2^(attempt-1), 10min)`，第 1 次 30s / 第 2 次 60s / ... - `claimable()` 过滤退避冷却中的任务 - `nowMs` 注入点（默认 `Date.now`）使测试可快进时钟验证退避行为 ### 3. needs_attention 一键重投 - `Task` 新增 `retryBaseline` 字段（默认 0）：记录上次重投时的失败 run 基线 - `Store.requeueTask(taskId)`：设置基线 = 当前失败 run 数 → 转 `queued`（仅限 needs_attention） - 编排器用净失败数（`allFailed - retryBaseline`）判断是否已耗尽重试次数 - API `POST /api/tasks/:id/requeue` + MCP `requeue_task` 工具 ### 4. 测试 - 重命名 `MAX_RETRIES` → `DEFAULT_MAX_RETRIES`，新增导出 `computeBackoffMs` - 新增测试（共 +18）：maxRetries=1/0、退避时序（精确 ms 边界）、重投后基线重置 - 迁移测试扩展：验证旧库补列后 maxRetries/timeoutMs/retryBaseline 使用默认值 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-13 04:04:41 +08:00
parent bb6186902a
commit c89d6d129c
12 changed files with 347 additions and 31 deletions
@@ -1,7 +1,7 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
 import { Store } from '../src/store/index.js';
-import { createOrchestrator, MAX_RETRIES, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
+import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
 import type { RunnerResult } from '../src/executor/runner.js';
 import type { ReviewResult } from '../src/executor/reviewer.js';
 import type { Autonomy } from '../src/model/types.js';
@@ -39,10 +39,17 @@ function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
    runner: async () => okRun,
    reviewCode: async () => okReview,
    reviewSecurity: async () => okSecurity,
+    nowMs: Date.now,
    ...overrides,
  };
 }

+/** 可前进的测试时钟（用于退避相关测试） */
+function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
+  let t = initialMs;
+  return { nowMs: () => t, advance: (ms) => { t += ms; } };
+}
+
 function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
  const store = new Store(':memory:');
  const p = store.createProject({
@@ -290,32 +297,165 @@ test('verify 不过：按失败处理（run failed + 重新入队）', async ()
  store.close();
 });

-test(`失败重试：重试 ${MAX_RETRIES} 次后 → needs_attention（共 ${MAX_RETRIES + 1} 次失败 run）`, async () => {
+test(`失败重试：重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention（共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run）`, async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });

  let attempts = 0;
+  // 使用快进时钟跳过退避冷却（每次 tick 前将时钟推进 10min）
+  const clock = makeClock();
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
+    nowMs: clock.nowMs,
  }));

-  for (let i = 1; i <= MAX_RETRIES; i++) {
+  for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
+    clock.advance(10 * 60_000);            // 跳过退避冷却
    orch.tick();
    await orch.drain();
    assert.equal(store.getTask(t.id)!.status, 'queued', `第 ${i} 次失败后应重新入队`);
  }

+  clock.advance(10 * 60_000);             // 跳过最后一次退避
  orch.tick();                              // 最后一次重试也失败
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
-  assert.equal(attempts, MAX_RETRIES + 1);
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);

  const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
-  assert.equal(failed.length, MAX_RETRIES + 1);
+  assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);

+  clock.advance(10 * 60_000);
  orch.tick();                              // needs_attention 不会再被领取
  await orch.drain();
-  assert.equal(attempts, MAX_RETRIES + 1);
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
+  store.close();
+});
+
+test('项目级 maxRetries=1：只重试 1 次就 → needs_attention', async () => {
+  const store = new Store(':memory:');
+  const p = store.createProject({
+    name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
+    autonomy: 'auto-easy', maxRetries: 1,
+  });
+  const projectId = p.id;
+  const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });
+
+  let attempts = 0;
+  const clock = makeClock();
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 第 1 次执行失败 → 重新入队（还有 1 次重试机会）
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+
+  // 第 2 次失败 → 超过 maxRetries=1 → needs_attention（快进时钟跳过退避）
+  clock.advance(10 * 60_000);
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, 2);
+  store.close();
+});
+
+test('项目级 maxRetries=0：首次失败直接 needs_attention（不重试）', async () => {
+  const store = new Store(':memory:');
+  const p = store.createProject({
+    name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
+    autonomy: 'auto-easy', maxRetries: 0,
+  });
+  const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });
+
+  let attempts = 0;
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+  }));
+
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, 1);
+  store.close();
+});
+
+test('computeBackoffMs：指数退避公式，30s 基数，上限 10min', () => {
+  assert.equal(computeBackoffMs(1), 30_000);
+  assert.equal(computeBackoffMs(2), 60_000);
+  assert.equal(computeBackoffMs(3), 120_000);
+  assert.equal(computeBackoffMs(4), 240_000);
+  assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
+  assert.equal(computeBackoffMs(100), 10 * 60_000);
+});
+
+test('退避冷却：首次失败重入队后，退避期内不被领取；退避过期后正常领取', async () => {
+  const { store, projectId } = setup('auto-easy');
+  const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
+
+  let attempts = 0;
+  // 使用可控时钟：初始时间 0，退避 30s（attempt=1 → BACKOFF_BASE=30000ms）
+  const clock = makeClock(0);
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 第 1 次失败（t=0）→ 重入队 + 退避（backoffUntil = 0 + 30000 = 30000ms）
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+  assert.equal(attempts, 1);
+
+  // t=29999：退避期内，不被领取
+  clock.advance(29_999);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, 1, '退避期内不应再次执行');
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+
+  // t=30001：退避过期，任务应被重新领取
+  clock.advance(2);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, 2, '退避过期后应再次执行');
+  store.close();
+});
+
+test('requeueTask：needs_attention → queued，重置重试基线，再次允许重试', async () => {
+  const { store, projectId } = setup('auto-easy');
+  const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });
+
+  let attempts = 0;
+  const clock = makeClock();
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 耗尽默认重试次数 → needs_attention（每次需快进时钟跳过退避）
+  for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
+    clock.advance(10 * 60_000);
+    orch.tick();
+    await orch.drain();
+  }
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
+
+  // 手动重投：重置基线，转 queued
+  const requeued = store.requeueTask(t.id);
+  assert.equal(requeued.status, 'queued');
+  assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);
+
+  // 重投后编排器应能再次执行（快进时钟确保无退避阻拦）
+  clock.advance(10 * 60_000);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
+  // 第 1 次净失败后应重入队（基线已重置，净失败=1 < maxRetries=2）
+  assert.equal(store.getTask(t.id)!.status, 'queued');
  store.close();
 });