merge: maestro/tsk_N5wO3Armums2 [重试与超时策略细化(可配 + 退避)]

# Conflicts: # src/api/server.ts # test/store.test.ts
2026-06-13 10:20:20 +08:00
parent d56b0641c5 c89d6d129c
commit a00091329d
12 changed files with 347 additions and 31 deletions
@@ -1,7 +1,7 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
 import { Store } from '../src/store/index.js';
-import { createOrchestrator, MAX_RETRIES, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
+import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
 import type { RunnerResult } from '../src/executor/runner.js';
 import type { ReviewResult } from '../src/executor/reviewer.js';
 import type { Autonomy } from '../src/model/types.js';
@@ -39,10 +39,17 @@ function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
    runner: async () => okRun,
    reviewCode: async () => okReview,
    reviewSecurity: async () => okSecurity,
+    nowMs: Date.now,
    ...overrides,
  };
 }

+/** 可前进的测试时钟（用于退避相关测试） */
+function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
+  let t = initialMs;
+  return { nowMs: () => t, advance: (ms) => { t += ms; } };
+}
+
 function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
  const store = new Store(':memory:');
  const p = store.createProject({
@@ -291,32 +298,165 @@ test('verify 不过：按失败处理（run failed + 重新入队）', async ()
  store.close();
 });

-test(`失败重试：重试 ${MAX_RETRIES} 次后 → needs_attention（共 ${MAX_RETRIES + 1} 次失败 run）`, async () => {
+test(`失败重试：重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention（共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run）`, async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });

  let attempts = 0;
+  // 使用快进时钟跳过退避冷却（每次 tick 前将时钟推进 10min）
+  const clock = makeClock();
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
+    nowMs: clock.nowMs,
  }));

-  for (let i = 1; i <= MAX_RETRIES; i++) {
+  for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
+    clock.advance(10 * 60_000);            // 跳过退避冷却
    orch.tick();
    await orch.drain();
    assert.equal(store.getTask(t.id)!.status, 'queued', `第 ${i} 次失败后应重新入队`);
  }

+  clock.advance(10 * 60_000);             // 跳过最后一次退避
  orch.tick();                              // 最后一次重试也失败
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
-  assert.equal(attempts, MAX_RETRIES + 1);
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);

  const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
-  assert.equal(failed.length, MAX_RETRIES + 1);
+  assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);

+  clock.advance(10 * 60_000);
  orch.tick();                              // needs_attention 不会再被领取
  await orch.drain();
-  assert.equal(attempts, MAX_RETRIES + 1);
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
+  store.close();
+});
+
+test('项目级 maxRetries=1：只重试 1 次就 → needs_attention', async () => {
+  const store = new Store(':memory:');
+  const p = store.createProject({
+    name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
+    autonomy: 'auto-easy', maxRetries: 1,
+  });
+  const projectId = p.id;
+  const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });
+
+  let attempts = 0;
+  const clock = makeClock();
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 第 1 次执行失败 → 重新入队（还有 1 次重试机会）
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+
+  // 第 2 次失败 → 超过 maxRetries=1 → needs_attention（快进时钟跳过退避）
+  clock.advance(10 * 60_000);
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, 2);
+  store.close();
+});
+
+test('项目级 maxRetries=0：首次失败直接 needs_attention（不重试）', async () => {
+  const store = new Store(':memory:');
+  const p = store.createProject({
+    name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
+    autonomy: 'auto-easy', maxRetries: 0,
+  });
+  const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });
+
+  let attempts = 0;
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+  }));
+
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, 1);
+  store.close();
+});
+
+test('computeBackoffMs：指数退避公式，30s 基数，上限 10min', () => {
+  assert.equal(computeBackoffMs(1), 30_000);
+  assert.equal(computeBackoffMs(2), 60_000);
+  assert.equal(computeBackoffMs(3), 120_000);
+  assert.equal(computeBackoffMs(4), 240_000);
+  assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
+  assert.equal(computeBackoffMs(100), 10 * 60_000);
+});
+
+test('退避冷却：首次失败重入队后，退避期内不被领取；退避过期后正常领取', async () => {
+  const { store, projectId } = setup('auto-easy');
+  const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
+
+  let attempts = 0;
+  // 使用可控时钟：初始时间 0，退避 30s（attempt=1 → BACKOFF_BASE=30000ms）
+  const clock = makeClock(0);
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 第 1 次失败（t=0）→ 重入队 + 退避（backoffUntil = 0 + 30000 = 30000ms）
+  orch.tick();
+  await orch.drain();
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+  assert.equal(attempts, 1);
+
+  // t=29999：退避期内，不被领取
+  clock.advance(29_999);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, 1, '退避期内不应再次执行');
+  assert.equal(store.getTask(t.id)!.status, 'queued');
+
+  // t=30001：退避过期，任务应被重新领取
+  clock.advance(2);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, 2, '退避过期后应再次执行');
+  store.close();
+});
+
+test('requeueTask：needs_attention → queued，重置重试基线，再次允许重试', async () => {
+  const { store, projectId } = setup('auto-easy');
+  const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });
+
+  let attempts = 0;
+  const clock = makeClock();
+  const orch = createOrchestrator(store, noopLog, mockDeps({
+    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
+    nowMs: clock.nowMs,
+  }));
+
+  // 耗尽默认重试次数 → needs_attention（每次需快进时钟跳过退避）
+  for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
+    clock.advance(10 * 60_000);
+    orch.tick();
+    await orch.drain();
+  }
+  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
+
+  // 手动重投：重置基线，转 queued
+  const requeued = store.requeueTask(t.id);
+  assert.equal(requeued.status, 'queued');
+  assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);
+
+  // 重投后编排器应能再次执行（快进时钟确保无退避阻拦）
+  clock.advance(10 * 60_000);
+  orch.tick();
+  await orch.drain();
+  assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
+  // 第 1 次净失败后应重入队（基线已重置，净失败=1 < maxRetries=2）
+  assert.equal(store.getTask(t.id)!.status, 'queued');
  store.close();
 });