merge: maestro/tsk_N5wO3Armums2 [重试与超时策略细化(可配 + 退避)]

# Conflicts:
#	src/api/server.ts
#	test/store.test.ts
This commit is contained in:
wangjia
2026-06-13 10:20:20 +08:00
12 changed files with 347 additions and 31 deletions
+146 -6
View File
@@ -1,7 +1,7 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { Store } from '../src/store/index.js';
import { createOrchestrator, MAX_RETRIES, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
import type { RunnerResult } from '../src/executor/runner.js';
import type { ReviewResult } from '../src/executor/reviewer.js';
import type { Autonomy } from '../src/model/types.js';
@@ -39,10 +39,17 @@ function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
runner: async () => okRun,
reviewCode: async () => okReview,
reviewSecurity: async () => okSecurity,
nowMs: Date.now,
...overrides,
};
}
/** 可前进的测试时钟(用于退避相关测试) */
function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
let t = initialMs;
return { nowMs: () => t, advance: (ms) => { t += ms; } };
}
function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
const store = new Store(':memory:');
const p = store.createProject({
@@ -291,32 +298,165 @@ test('verify 不过:按失败处理(run failed + 重新入队)', async ()
store.close();
});
test(`失败重试:重试 ${MAX_RETRIES} 次后 → needs_attention(共 ${MAX_RETRIES + 1} 次失败 run`, async () => {
test(`失败重试:重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention(共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run`, async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });
let attempts = 0;
// 使用快进时钟跳过退避冷却(每次 tick 前将时钟推进 10min
const clock = makeClock();
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
nowMs: clock.nowMs,
}));
for (let i = 1; i <= MAX_RETRIES; i++) {
for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
clock.advance(10 * 60_000); // 跳过退避冷却
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued', `${i} 次失败后应重新入队`);
}
clock.advance(10 * 60_000); // 跳过最后一次退避
orch.tick(); // 最后一次重试也失败
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, MAX_RETRIES + 1);
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
assert.equal(failed.length, MAX_RETRIES + 1);
assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);
clock.advance(10 * 60_000);
orch.tick(); // needs_attention 不会再被领取
await orch.drain();
assert.equal(attempts, MAX_RETRIES + 1);
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
store.close();
});
test('项目级 maxRetries=1:只重试 1 次就 → needs_attention', async () => {
const store = new Store(':memory:');
const p = store.createProject({
name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
autonomy: 'auto-easy', maxRetries: 1,
});
const projectId = p.id;
const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });
let attempts = 0;
const clock = makeClock();
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
nowMs: clock.nowMs,
}));
// 第 1 次执行失败 → 重新入队(还有 1 次重试机会)
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued');
// 第 2 次失败 → 超过 maxRetries=1 → needs_attention(快进时钟跳过退避)
clock.advance(10 * 60_000);
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, 2);
store.close();
});
test('项目级 maxRetries=0:首次失败直接 needs_attention(不重试)', async () => {
const store = new Store(':memory:');
const p = store.createProject({
name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
autonomy: 'auto-easy', maxRetries: 0,
});
const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });
let attempts = 0;
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
}));
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, 1);
store.close();
});
test('computeBackoffMs:指数退避公式,30s 基数,上限 10min', () => {
assert.equal(computeBackoffMs(1), 30_000);
assert.equal(computeBackoffMs(2), 60_000);
assert.equal(computeBackoffMs(3), 120_000);
assert.equal(computeBackoffMs(4), 240_000);
assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
assert.equal(computeBackoffMs(100), 10 * 60_000);
});
test('退避冷却:首次失败重入队后,退避期内不被领取;退避过期后正常领取', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
let attempts = 0;
// 使用可控时钟:初始时间 0,退避 30sattempt=1 → BACKOFF_BASE=30000ms
const clock = makeClock(0);
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
nowMs: clock.nowMs,
}));
// 第 1 次失败(t=0)→ 重入队 + 退避(backoffUntil = 0 + 30000 = 30000ms
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued');
assert.equal(attempts, 1);
// t=29999:退避期内,不被领取
clock.advance(29_999);
orch.tick();
await orch.drain();
assert.equal(attempts, 1, '退避期内不应再次执行');
assert.equal(store.getTask(t.id)!.status, 'queued');
// t=30001:退避过期,任务应被重新领取
clock.advance(2);
orch.tick();
await orch.drain();
assert.equal(attempts, 2, '退避过期后应再次执行');
store.close();
});
test('requeueTaskneeds_attention → queued,重置重试基线,再次允许重试', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });
let attempts = 0;
const clock = makeClock();
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
nowMs: clock.nowMs,
}));
// 耗尽默认重试次数 → needs_attention(每次需快进时钟跳过退避)
for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
clock.advance(10 * 60_000);
orch.tick();
await orch.drain();
}
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
// 手动重投:重置基线,转 queued
const requeued = store.requeueTask(t.id);
assert.equal(requeued.status, 'queued');
assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);
// 重投后编排器应能再次执行(快进时钟确保无退避阻拦)
clock.advance(10 * 60_000);
orch.tick();
await orch.drain();
assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
// 第 1 次净失败后应重入队(基线已重置,净失败=1 < maxRetries=2
assert.equal(store.getTask(t.id)!.status, 'queued');
store.close();
});
+66
View File
@@ -301,3 +301,69 @@ test('合并失败补救:建最高优先级 easy 任务,且幂等不重复
assert.notEqual(fresh.id, rem.id, '旧补救任务已结束 → 建新的');
s.close();
});
test('patchProjectmaxRetries/timeoutMs 可配 + 校验', () => {
const s = freshStore();
const p = s.createProject({ name: 'cfg', repoPath: '/tmp/cfg-' + Math.random() });
// 默认值
assert.equal(p.maxRetries, 2);
assert.equal(p.timeoutMs, 1_800_000);
// 更新 maxRetries + timeoutMs
const updated = s.patchProject(p.id, { maxRetries: 5, timeoutMs: 60_000 });
assert.equal(updated.maxRetries, 5);
assert.equal(updated.timeoutMs, 60_000);
// 校验:maxRetries 必须 >=0 的整数
assert.throws(() => s.patchProject(p.id, { maxRetries: -1 }), /maxRetries/);
assert.throws(() => s.patchProject(p.id, { maxRetries: 1.5 }), /maxRetries/);
// 校验:timeoutMs 必须 >=1000
assert.throws(() => s.patchProject(p.id, { timeoutMs: 500 }), /timeoutMs/);
s.close();
});
test('createProjectmaxRetries/timeoutMs 自定义初值', () => {
const s = freshStore();
const p = s.createProject({ name: 'custom', repoPath: '/tmp/custom-' + Math.random(), maxRetries: 0, timeoutMs: 120_000 });
assert.equal(p.maxRetries, 0);
assert.equal(p.timeoutMs, 120_000);
s.close();
});
test('requeueTaskneeds_attention → queued,重置 retryBaseline', () => {
const s = freshStore();
const p = s.createProject({ name: 'rq', repoPath: '/tmp/rq-' + Math.random() });
const t = s.createTask({ projectId: p.id, title: 'flaky', complexity: 'easy' });
// 推进到 needs_attention(模拟 3 次失败 run
s.transition(t.id, 'queued');
s.transition(t.id, 'executing');
s.transition(t.id, 'failed');
s.transition(t.id, 'queued');
s.transition(t.id, 'executing');
s.transition(t.id, 'failed');
s.transition(t.id, 'queued');
s.transition(t.id, 'executing');
s.transition(t.id, 'failed');
s.transition(t.id, 'needs_attention');
// 模拟 3 条 failed executor run
const r1 = s.startRun(t.id, 'executor');
s.finishRun(r1.id, 'failed', { error: 'boom1' });
const r2 = s.startRun(t.id, 'executor');
s.finishRun(r2.id, 'failed', { error: 'boom2' });
const r3 = s.startRun(t.id, 'executor');
s.finishRun(r3.id, 'failed', { error: 'boom3' });
assert.equal(s.getTask(t.id)!.retryBaseline, 0);
// 一键重投
const requeued = s.requeueTask(t.id);
assert.equal(requeued.status, 'queued');
assert.equal(requeued.retryBaseline, 3); // 基线设为当前失败 run 数
// 只有 needs_attention 状态才可重投
assert.throws(() => s.requeueTask(t.id), /needs_attention/);
s.close();
});
+7 -1
View File
@@ -169,17 +169,23 @@ test('迁移:旧版库(无 source_ref/last_sync_at)打开后补列且数
const projCols = (db.prepare(`PRAGMA table_info(projects)`).all() as Array<{ name: string }>).map((c) => c.name);
const taskCols = (db.prepare(`PRAGMA table_info(tasks)`).all() as Array<{ name: string }>).map((c) => c.name);
assert.ok(projCols.includes('last_sync_at'));
assert.ok(projCols.includes('max_retries'));
assert.ok(projCols.includes('timeout_ms'));
assert.ok(taskCols.includes('source_ref'));
assert.ok(taskCols.includes('retry_baseline'));
db.close();
// Store 能正常读旧数据,新字段为 null
// Store 能正常读旧数据,新字段使用默认值
const s = new Store(file);
const projects = s.listProjects();
assert.equal(projects.length, 1);
assert.equal(projects[0].name, '旧项目');
assert.equal(projects[0].lastSyncAt, null);
assert.equal(projects[0].maxRetries, 2); // 默认值
assert.equal(projects[0].timeoutMs, 1_800_000); // 默认 30min
const t = s.getTask('tsk_old');
assert.equal(t?.title, '旧任务');
assert.equal(t?.retryBaseline, 0); // 默认基线
// 新方法在迁移后的旧库上可用
s.setSourceRef('tsk_old', 'todo:99');
assert.equal(s.getTaskBySourceRef('prj_old', 'todo:99')?.id, 'tsk_old');