merge phase3-C: daemon 侧(监工 + ingest + 接线 + 测试)

This commit is contained in:
wangjia
2026-06-13 13:02:13 +08:00
5 changed files with 765 additions and 567 deletions
+205
View File
@@ -0,0 +1,205 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Store } from '../src/store/index.js';
import { ingestRun } from '../src/daemon/ingest.js';
import { appendOutbox, type OutboxPayload } from '../src/executor/protocol.js';
import { branchFor, worktreeDirFor } from '../src/executor/worktree.js';
const noopLog = { info: (): void => undefined, error: (): void => undefined };
/**
* 隔离的 MAESTRO_DATA_DIRoutbox 落在临时目录),跑回调,结束清理。
* protocol.ts 的 runDir/outboxPath 在调用时读 env,故进 cb 前设好、cb 内的 append/ingest 都命中临时目录。
*/
function withTmpDataDir(cb: () => void): void {
const dir = mkdtempSync(join(tmpdir(), 'maestro-ingest-'));
const prev = process.env.MAESTRO_DATA_DIR;
process.env.MAESTRO_DATA_DIR = dir;
try {
cb();
} finally {
if (prev === undefined) delete process.env.MAESTRO_DATA_DIR;
else process.env.MAESTRO_DATA_DIR = prev;
rmSync(dir, { recursive: true, force: true });
}
}
/** 真 Store + 项目 + easy 任务推进到 executing,并建一条 executor run(模拟 daemon 已领取、worker 已 spawn)。 */
function setupExecuting(opts: { autonomy?: 'auto-easy' | 'auto-approved'; maxRetries?: number } = {}): {
store: Store; projectId: string; taskId: string; runId: string; dir: string; branch: string;
} {
const store = new Store(':memory:');
const p = store.createProject({
name: 'ingest', repoPath: '/tmp/ingest-repo-' + Math.random(),
autonomy: opts.autonomy ?? 'auto-easy', maxRetries: opts.maxRetries,
});
const t = store.createTask({ projectId: p.id, title: 'tweak', complexity: 'easy' });
store.setOperations(t.id, '在 README.md 追加一行');
const branch = branchFor(t.id);
const dir = worktreeDirFor(p.repoPath, t.id);
store.transition(t.id, 'queued', { by: 'test' });
store.transition(t.id, 'executing', { by: 'test' });
const run = store.startRun(t.id, 'executor', { worktree: dir, branch });
store.setWorkerPid(run.id, 4242);
return { store, projectId: p.id, taskId: t.id, runId: run.id, dir, branch };
}
function resultPayload(branch: string, worktree: string): OutboxPayload {
return {
type: 'result',
branch, worktree,
diffSummary: ' README.md | 1 +',
commits: ['abc1234 hello maestro'],
executor: { transcriptRef: '/tmp/exec.jsonl', sessionId: 'sess-exec-1' },
code: { summary: '## 做了什么\nmock 复审通过', verdict: 'approve', transcriptRef: '/tmp/code.jsonl' },
security: { summary: '## 安全审计\nmock 审计通过', verdict: 'approve', transcriptRef: '/tmp/sec.jsonl' },
};
}
test('ingest resulttask→exec_reviewsetResult 四字段正确,reviewer/security/executor 各 1 条 succeeded', () => {
withTmpDataDir(() => {
const { store, taskId, runId, dir, branch } = setupExecuting();
appendOutbox(runId, { type: 'started', pid: 4242, worktree: dir, branch, model: 'claude-sonnet-4-6' });
appendOutbox(runId, { type: 'phase', phase: 'executing' });
appendOutbox(runId, resultPayload(branch, dir));
appendOutbox(runId, { type: 'done' });
ingestRun(store, noopLog, runId);
const done = store.getTask(taskId)!;
assert.equal(done.status, 'exec_review');
assert.deepEqual(done.result, {
branch,
worktree: dir,
diffSummary: ' README.md | 1 +',
commits: ['abc1234 hello maestro'],
prUrl: null,
summary: '## 做了什么\nmock 复审通过',
verdict: 'approve',
securitySummary: '## 安全审计\nmock 审计通过',
securityVerdict: 'approve',
mergeTaskId: null,
});
const runs = store.listRuns(taskId);
const executor = runs.find((r) => r.kind === 'executor')!;
assert.equal(executor.status, 'succeeded');
assert.equal(executor.transcriptRef, '/tmp/exec.jsonl');
assert.equal(executor.claudeSessionId, 'sess-exec-1');
const reviewer = runs.filter((r) => r.kind === 'reviewer');
assert.equal(reviewer.length, 1);
assert.equal(reviewer[0].status, 'succeeded');
assert.equal(reviewer[0].transcriptRef, '/tmp/code.jsonl');
const security = runs.filter((r) => r.kind === 'security');
assert.equal(security.length, 1);
assert.equal(security[0].status, 'succeeded');
assert.equal(security[0].transcriptRef, '/tmp/sec.jsonl');
// 游标推进到末条(done.seq=4
assert.equal(store.getRun(runId)!.lastSeq, 4);
store.close();
});
});
test('ingest result:任一 verdict=reject 也照常落进 result(裁决归用户)', () => {
withTmpDataDir(() => {
const { store, taskId, runId, dir, branch } = setupExecuting();
appendOutbox(runId, {
...resultPayload(branch, dir),
code: { summary: '发现问题', verdict: 'reject', transcriptRef: '/tmp/code.jsonl' },
security: { summary: '发现密钥泄露', verdict: 'reject', transcriptRef: '/tmp/sec.jsonl' },
} as OutboxPayload);
ingestRun(store, noopLog, runId);
const done = store.getTask(taskId)!;
assert.equal(done.status, 'exec_review');
assert.equal(done.result!.verdict, 'reject');
assert.equal(done.result!.summary, '发现问题');
assert.equal(done.result!.securityVerdict, 'reject');
assert.equal(done.result!.securitySummary, '发现密钥泄露');
store.close();
});
});
test('ingest failed:首次失败 → task 转 queued + 持久化退避(nextEligibleAt 有值)+ executor run failed', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupExecuting();
appendOutbox(runId, { type: 'started', pid: 4242, worktree: '/x', branch: 'b', model: 'm' });
appendOutbox(runId, { type: 'failed', error: 'boom #1', transcriptRef: '/tmp/f.jsonl', sessionId: 'sess-f-1' });
appendOutbox(runId, { type: 'done' });
ingestRun(store, noopLog, runId);
const t = store.getTask(taskId)!;
assert.equal(t.status, 'queued'); // 第一次失败 → 重入队(默认 maxRetries=2
assert.ok(t.nextEligibleAt, 'nextEligibleAt 应被持久化(退避)');
assert.ok(Date.parse(t.nextEligibleAt!) > Date.now() - 1000, 'nextEligibleAt 在未来');
const executor = store.listRuns(taskId).find((r) => r.kind === 'executor')!;
assert.equal(executor.status, 'failed');
assert.match(executor.error ?? '', /boom #1/);
assert.equal(executor.transcriptRef, '/tmp/f.jsonl');
assert.equal(executor.claudeSessionId, 'sess-f-1');
store.close();
});
});
test('ingest failedmaxRetries=0 → 首次失败直接 needs_attention', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupExecuting({ maxRetries: 0 });
appendOutbox(runId, { type: 'failed', error: 'boom', transcriptRef: null, sessionId: null });
ingestRun(store, noopLog, runId);
assert.equal(store.getTask(taskId)!.status, 'needs_attention');
store.close();
});
});
test('幂等:同一批 outbox ingest 两次,DB 不重复变更(lastSeq 生效)', () => {
withTmpDataDir(() => {
const { store, taskId, runId, dir, branch } = setupExecuting();
appendOutbox(runId, { type: 'started', pid: 4242, worktree: dir, branch, model: 'm' });
appendOutbox(runId, resultPayload(branch, dir));
appendOutbox(runId, { type: 'done' });
ingestRun(store, noopLog, runId);
const after1 = store.getTask(taskId)!;
const runs1 = store.listRuns(taskId);
assert.equal(after1.status, 'exec_review');
assert.equal(runs1.filter((r) => r.kind === 'reviewer').length, 1);
assert.equal(runs1.filter((r) => r.kind === 'security').length, 1);
const lastSeq1 = store.getRun(runId)!.lastSeq;
// 第二次 ingestseq 全部 ≤ lastSeq → 不处理任何记录
ingestRun(store, noopLog, runId);
const runs2 = store.listRuns(taskId);
assert.equal(runs2.length, runs1.length, '复审 run 不应重复创建');
assert.equal(runs2.filter((r) => r.kind === 'reviewer').length, 1);
assert.equal(runs2.filter((r) => r.kind === 'security').length, 1);
assert.equal(store.getRun(runId)!.lastSeq, lastSeq1, 'lastSeq 不变');
assert.equal(store.getTask(taskId)!.status, 'exec_review');
store.close();
});
});
test('ingest 续读:先 ingest 半截(started/phase),再追加 result,第二次 ingest 完成 exec_review', () => {
withTmpDataDir(() => {
const { store, taskId, runId, dir, branch } = setupExecuting();
appendOutbox(runId, { type: 'started', pid: 4242, worktree: dir, branch, model: 'm' });
appendOutbox(runId, { type: 'phase', phase: 'executing' });
ingestRun(store, noopLog, runId);
assert.equal(store.getTask(taskId)!.status, 'executing'); // 还没到 result,仍 executing
assert.equal(store.getRun(runId)!.lastSeq, 2);
appendOutbox(runId, resultPayload(branch, dir));
appendOutbox(runId, { type: 'done' });
ingestRun(store, noopLog, runId);
assert.equal(store.getTask(taskId)!.status, 'exec_review');
assert.equal(store.getRun(runId)!.lastSeq, 4);
store.close();
});
});
+292 -398
View File
@@ -1,53 +1,44 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { Store } from '../src/store/index.js';
import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
import type { RunnerResult } from '../src/executor/runner.js';
import type { ReviewResult } from '../src/executor/reviewer.js';
import {
createOrchestrator, workerEntry, workerEnv, DEFAULT_MAX_RETRIES,
type OrchestratorDeps,
} from '../src/daemon/orchestrator.js';
import type { JobSpec } from '../src/executor/protocol.js';
import type { Autonomy } from '../src/model/types.js';
const noopLog = { info: (): void => undefined, error: (): void => undefined };
function deferred<T>(): { promise: Promise<T>; resolve: (v: T) => void } {
let resolve!: (v: T) => void;
const promise = new Promise<T>((r) => { resolve = r; });
return { promise, resolve };
/** 可前进的测试时钟。 */
function makeClock(initialMs = Date.now()): { nowMs: () => number; advance: (ms: number) => void } {
let t = initialMs;
return { nowMs: () => t, advance: (ms) => { t += ms; } };
}
const okRun: RunnerResult = { ok: true, transcriptRef: '/tmp/fake.jsonl', sessionId: 'sess-mock-1', finalText: '执行自述:改了 README' };
interface MockState {
spawned: string[]; // 被 spawn 的 runId 列表
jobs: JobSpec[]; // 被 writeJobSpec 的 job 列表
ingestCalls: number;
alive: boolean; // isWorkerAlive 的返回(reaper 用)
nextPid: number;
}
const okReview: ReviewResult = {
summary: '## 做了什么\nmock 复审通过',
verdict: 'approve',
transcriptRef: '/tmp/fake-review.jsonl',
sessionId: 'sess-review-1',
};
const okSecurity: ReviewResult = {
summary: '## 安全审计\nmock 审计通过',
verdict: 'approve',
transcriptRef: '/tmp/fake-security.jsonl',
sessionId: 'sess-security-1',
};
/** 全 mock 依赖(不真起 CC、不动 git):可按用例覆盖 */
function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
return {
createWorktree: async (_repo, taskId) => ({ dir: `/tmp/fake-wt/${taskId}`, branch: `maestro/${taskId}` }),
worktreeDiff: async () => ({ diffSummary: ' README.md | 1 +', commits: ['abc1234 hello maestro'] }),
verify: async () => ({ ok: true, exitCode: 0, logRef: null }),
runner: async () => okRun,
reviewCode: async () => okReview,
reviewSecurity: async () => okSecurity,
/** 全 mock 依赖:不真 spawn / 不真判活 / ingest no-op / job 不落盘。可按用例覆盖。 */
function mockDeps(state: MockState, overrides: Partial<OrchestratorDeps> = {}): { deps: OrchestratorDeps } {
const deps: OrchestratorDeps = {
spawnWorker: (runId: string): number => { state.spawned.push(runId); return state.nextPid++; },
isWorkerAlive: () => state.alive,
ingestAll: () => { state.ingestCalls++; },
writeJobSpec: (job: JobSpec) => { state.jobs.push(job); },
nowMs: Date.now,
...overrides,
};
return { deps };
}
/** 可前进的测试时钟(用于退避相关测试) */
function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
let t = initialMs;
return { nowMs: () => t, advance: (ms) => { t += ms; } };
function freshState(): MockState {
return { spawned: [], jobs: [], ingestCalls: 0, alive: true, nextPid: 1000 };
}
function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
@@ -58,25 +49,139 @@ function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId:
return { store, projectId: p.id };
}
async function settle(): Promise<void> {
await new Promise((r) => setImmediate(r));
}
// ───────────────────────── workerEntry / workerEnv(可读测的小函数)─────────────────────────
test('autonomy=manual:编排器不领取任何任务', async () => {
const { store, projectId } = setup('manual');
const t = store.createTask({ projectId, title: 'easy task', complexity: 'easy' });
let calls = 0;
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { calls++; return okRun; },
}));
test('workerEntry:默认指向 ../executor/worker.jsnode 运行)', () => {
const prev = process.env.MAESTRO_WORKER_CMD;
delete process.env.MAESTRO_WORKER_CMD;
const [cmd, scriptPath] = workerEntry();
assert.equal(cmd, 'node');
assert.match(scriptPath, /executor[/\\]worker\.js$/);
if (prev !== undefined) process.env.MAESTRO_WORKER_CMD = prev;
});
test('workerEntryMAESTRO_WORKER_CMD 覆盖整条命令(空格分隔)', () => {
const prev = process.env.MAESTRO_WORKER_CMD;
process.env.MAESTRO_WORKER_CMD = 'npx tsx src/executor/worker.ts';
assert.deepEqual(workerEntry(), ['npx', 'tsx', 'src/executor/worker.ts']);
if (prev === undefined) delete process.env.MAESTRO_WORKER_CMD;
else process.env.MAESTRO_WORKER_CMD = prev;
});
test('workerEnv:只透传 PATH/HOME/LANG + ANTHROPIC_/CLAUDE_ 前缀;丢弃无关与 MAESTRO_ 控制变量', () => {
const env = workerEnv({
PATH: '/usr/bin', HOME: '/home/u', LANG: 'en_US.UTF-8',
ANTHROPIC_API_KEY: 'sk-x', CLAUDE_CODE_FOO: 'y',
MAESTRO_ORCH_INTERVAL: '15', MAESTRO_DATA_DIR: '/data',
SOME_SECRET: 'leak', AWS_SECRET_ACCESS_KEY: 'nope',
});
assert.equal(env.PATH, '/usr/bin');
assert.equal(env.HOME, '/home/u');
assert.equal(env.LANG, 'en_US.UTF-8');
assert.equal(env.ANTHROPIC_API_KEY, 'sk-x');
assert.equal(env.CLAUDE_CODE_FOO, 'y');
assert.equal(env.MAESTRO_DATA_DIR, '/data'); // 例外:worker 须看同一 runs/ 目录
assert.equal(env.MAESTRO_ORCH_INTERVAL, undefined, 'daemon 控制变量不下传');
assert.equal(env.SOME_SECRET, undefined, '无关变量不下传');
assert.equal(env.AWS_SECRET_ACCESS_KEY, undefined, '无关密钥不下传');
});
// ───────────────────────── tick 顺序:ingest + reaper 先于领取 ─────────────────────────
test('tick:每轮先 ingest 再 reaper 再领取', () => {
const { store, projectId } = setup('auto-easy');
store.createTask({ projectId, title: 'a', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state);
const orch = createOrchestrator(store, noopLog, deps);
orch.tick();
await orch.drain();
assert.equal(calls, 0);
assert.equal(state.ingestCalls, 1, '每轮 ingest 一次');
assert.equal(state.spawned.length, 1, '领取并 spawn 一个');
});
// ───────────────────────── 领取(监工)─────────────────────────
test('autonomy=manual:不领取', () => {
const { store, projectId } = setup('manual');
const t = store.createTask({ projectId, title: 'easy', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state);
createOrchestrator(store, noopLog, deps).tick();
assert.equal(state.spawned.length, 0);
assert.equal(store.getTask(t.id)!.status, 'ready');
store.close();
});
test('autonomy=auto-easy:只领 easymedium ready 不动', async () => {
test('project paused:不领取', () => {
const { store, projectId } = setup('auto-approved');
const t = store.createTask({ projectId, title: 'x', complexity: 'easy' });
store.patchProject(projectId, { status: 'paused' });
const state = freshState();
const { deps } = mockDeps(state);
createOrchestrator(store, noopLog, deps).tick();
assert.equal(state.spawned.length, 0);
assert.equal(store.getTask(t.id)!.status, 'ready');
store.close();
});
test('并发闸 concurrency=1:一轮只领一个;领取后 executing + 建 executor run + setWorkerPid + writeJobSpec', () => {
const { store, projectId } = setup('auto-approved', 1);
const t1 = store.createTask({ projectId, title: 'a', complexity: 'easy' });
const t2 = store.createTask({ projectId, title: 'b', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state);
const orch = createOrchestrator(store, noopLog, deps);
orch.claimTick();
// 并发=1:只领一个
assert.equal(state.spawned.length, 1);
assert.equal(state.jobs.length, 1);
// 被领的那个进 executing,另一个仍 ready
const a = store.getTask(t1.id)!;
const b = store.getTask(t2.id)!;
const executing = a.status === 'executing' ? a : b;
const stillReady = a.status === 'executing' ? b : a;
assert.equal(executing.status, 'executing');
assert.equal(stillReady.status, 'ready');
// 建了 executor run + setWorkerPid + writeJobSpec 内容正确
const runs = store.listRuns(executing.id);
assert.equal(runs.length, 1);
const run = runs[0];
assert.equal(run.kind, 'executor');
assert.equal(run.status, 'started');
assert.equal(run.branch, `maestro/${executing.id}`);
assert.ok(run.workerPid, 'setWorkerPid 已写 pid');
assert.equal(state.spawned[0], run.id, 'spawn 的 runId == 新建 executor run');
const job = state.jobs[0];
assert.equal(job.runId, run.id);
assert.equal(job.task.id, executing.id);
assert.equal(job.task.status, 'executing');
assert.equal(job.branch, `maestro/${executing.id}`);
assert.equal(job.worktreeDir, run.worktree);
// 槽位占满 → 下一轮不再领(仍只 spawn 过一个)
orch.claimTick();
assert.equal(state.spawned.length, 1, '并发=1executing 占满 → 不再领');
store.close();
});
test('并发 concurrency=2:一轮领满两个', () => {
const { store, projectId } = setup('auto-approved', 2);
store.createTask({ projectId, title: 'a', complexity: 'easy' });
store.createTask({ projectId, title: 'b', complexity: 'easy' });
store.createTask({ projectId, title: 'c', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state);
createOrchestrator(store, noopLog, deps).claimTick();
assert.equal(state.spawned.length, 2, '并发=2 一轮领两个');
assert.equal(store.listTasks(projectId).filter((t) => t.status === 'executing').length, 2);
store.close();
});
test('auto-easy:只领 easymedium ready 不动', () => {
const { store, projectId } = setup('auto-easy', 5);
const easy = store.createTask({ projectId, title: 'small', complexity: 'easy' });
const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
@@ -85,392 +190,181 @@ test('autonomy=auto-easy:只领 easymedium ready 不动', async () => {
store.decide(medium.id, 'accept', 'user'); // medium → ready
assert.equal(store.getTask(medium.id)!.status, 'ready');
const ran: string[] = [];
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async (task) => { ran.push(task.id); return okRun; },
}));
orch.tick();
await orch.drain();
const state = freshState();
const { deps } = mockDeps(state);
createOrchestrator(store, noopLog, deps).claimTick();
assert.deepEqual(ran, [easy.id]);
assert.equal(store.getTask(easy.id)!.status, 'exec_review');
assert.equal(store.getTask(medium.id)!.status, 'ready'); // 不自动跑
assert.equal(state.spawned.length, 1);
assert.equal(store.getTask(easy.id)!.status, 'executing');
assert.equal(store.getTask(medium.id)!.status, 'ready'); // 不领 medium
store.close();
});
test('autonomy=auto-approved:领全部 ready(含 medium依赖未满足/非叶子不领', async () => {
test('auto-approved:领 ready(含 medium依赖未满足/非叶子不领', () => {
const { store, projectId } = setup('auto-approved', 5);
const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
store.setSpec(medium.id, '方案');
store.transition(medium.id, 'spec_review');
store.decide(medium.id, 'accept', 'user');
// 依赖未 done 的任务:建在 medium 上 → blocked,不可领
const dep = store.createTask({ projectId, title: 'after-mid', complexity: 'easy', deps: [medium.id] });
assert.equal(store.getTask(dep.id)!.status, 'blocked');
const ran: string[] = [];
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async (task) => { ran.push(task.id); return okRun; },
}));
orch.tick();
await orch.drain();
const state = freshState();
const { deps } = mockDeps(state);
createOrchestrator(store, noopLog, deps).claimTick();
assert.deepEqual(ran, [medium.id]);
assert.equal(store.getTask(medium.id)!.status, 'exec_review');
assert.equal(state.spawned.length, 1);
assert.equal(store.getTask(medium.id)!.status, 'executing');
assert.equal(store.getTask(dep.id)!.status, 'blocked');
store.close();
});
test('concurrency=1:同项目同轮只领 1 个,跑完下一轮再领', async () => {
const { store, projectId } = setup('auto-approved', 1);
const t1 = store.createTask({ projectId, title: 'a', complexity: 'easy' });
const t2 = store.createTask({ projectId, title: 'b', complexity: 'easy' });
const gate = deferred<void>();
const started: string[] = [];
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async (task) => { started.push(task.id); await gate.promise; return okRun; },
}));
orch.tick();
await settle();
assert.deepEqual(started, [t1.id], '并发=1 只应启动第一个任务');
assert.equal(store.getTask(t1.id)!.status, 'executing');
assert.equal(store.getTask(t2.id)!.status, 'ready');
assert.equal(orch.inflight.size, 1);
orch.tick(); // 在途占满 → 本轮不领
await settle();
assert.deepEqual(started, [t1.id]);
gate.resolve();
await orch.drain();
assert.equal(store.getTask(t1.id)!.status, 'exec_review');
orch.tick(); // 槽位释放 → 领第二个
gate.resolve();
await orch.drain();
assert.deepEqual(started, [t1.id, t2.id]);
assert.equal(store.getTask(t2.id)!.status, 'exec_review');
store.close();
});
test('成功路径:状态流转 + setResult(四字段) + executor/reviewer/security 三 run succeeded', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'tweak', complexity: 'easy' });
store.setOperations(t.id, '在 README.md 追加一行');
const seen: string[] = [];
store.subscribe((e) => { if (e.type === 'status.changed' && e.taskId === t.id) seen.push(String(e.payload.to)); });
let codeReportSeen: string | null = null;
let secReportSeen: string | null = null;
const orch = createOrchestrator(store, noopLog, mockDeps({
reviewCode: async (_task, _project, _wt, _runId, executorReport) => {
codeReportSeen = executorReport;
return okReview;
},
reviewSecurity: async (_task, _project, _wt, _runId, executorReport) => {
secReportSeen = executorReport;
return okSecurity;
},
}));
orch.tick();
await orch.drain();
const done = store.getTask(t.id)!;
assert.equal(done.status, 'exec_review');
assert.deepEqual(seen, ['queued', 'executing', 'exec_review']);
assert.deepEqual(done.result, {
branch: `maestro/${t.id}`,
worktree: `/tmp/fake-wt/${t.id}`,
diffSummary: ' README.md | 1 +',
commits: ['abc1234 hello maestro'],
prUrl: null,
summary: '## 做了什么\nmock 复审通过',
verdict: 'approve',
securitySummary: '## 安全审计\nmock 审计通过',
securityVerdict: 'approve',
mergeTaskId: null,
});
assert.equal(codeReportSeen, '执行自述:改了 README'); // runner finalText 传给两个复审作执行者自述
assert.equal(secReportSeen, '执行自述:改了 README');
const runs = store.listRuns(t.id);
assert.equal(runs.length, 3);
const executor = runs.find((r) => r.kind === 'executor')!;
assert.equal(executor.status, 'succeeded');
assert.equal(executor.branch, `maestro/${t.id}`);
assert.equal(executor.transcriptRef, '/tmp/fake.jsonl');
assert.equal(executor.claudeSessionId, 'sess-mock-1');
const reviewer = runs.find((r) => r.kind === 'reviewer')!;
assert.equal(reviewer.status, 'succeeded');
assert.equal(reviewer.transcriptRef, '/tmp/fake-review.jsonl');
assert.equal(reviewer.claudeSessionId, 'sess-review-1');
const security = runs.find((r) => r.kind === 'security')!;
assert.equal(security.status, 'succeeded');
assert.equal(security.transcriptRef, '/tmp/fake-security.jsonl');
assert.equal(security.claudeSessionId, 'sess-security-1');
store.close();
});
test('任一 verdict=reject 也照常落进 result(最终裁决仍归用户)', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'risky', complexity: 'easy' });
const orch = createOrchestrator(store, noopLog, mockDeps({
reviewCode: async () => ({ ...okReview, summary: '发现问题', verdict: 'reject' as const }),
reviewSecurity: async () => ({ ...okSecurity, summary: '发现密钥泄露', verdict: 'reject' as const }),
}));
orch.tick();
await orch.drain();
const done = store.getTask(t.id)!;
assert.equal(done.status, 'exec_review');
assert.equal(done.result!.verdict, 'reject');
assert.equal(done.result!.summary, '发现问题');
assert.equal(done.result!.securityVerdict, 'reject');
assert.equal(done.result!.securitySummary, '发现密钥泄露');
store.close();
});
test('code review 失败不挡任务:summary 记失败原因、verdict=null,安全审计照常跑', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'review-broken', complexity: 'easy' });
const orch = createOrchestrator(store, noopLog, mockDeps({
reviewCode: async () => { throw new Error('复审 CC 崩了'); },
}));
orch.tick();
await orch.drain();
const done = store.getTask(t.id)!;
assert.equal(done.status, 'exec_review'); // 不挡结果闸
assert.equal(done.result!.verdict, null);
assert.equal(done.result!.summary, '自动复审失败:复审 CC 崩了');
assert.equal(done.result!.securityVerdict, 'approve'); // 另一个复审不受影响
assert.equal(done.result!.securitySummary, '## 安全审计\nmock 审计通过');
const runs = store.listRuns(t.id);
assert.equal(runs.find((r) => r.kind === 'executor')!.status, 'succeeded');
const reviewer = runs.find((r) => r.kind === 'reviewer')!;
assert.equal(reviewer.status, 'failed');
assert.match(reviewer.error ?? '', /复审 CC 崩了/);
assert.equal(runs.find((r) => r.kind === 'security')!.status, 'succeeded');
store.close();
});
test('安全审计失败不挡任务:securitySummary 记失败原因、securityVerdict=nullcode review 不受影响', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'security-broken', complexity: 'easy' });
const orch = createOrchestrator(store, noopLog, mockDeps({
reviewSecurity: async () => { throw new Error('审计 CC 崩了'); },
}));
orch.tick();
await orch.drain();
const done = store.getTask(t.id)!;
assert.equal(done.status, 'exec_review');
assert.equal(done.result!.verdict, 'approve');
assert.equal(done.result!.summary, '## 做了什么\nmock 复审通过');
assert.equal(done.result!.securityVerdict, null);
assert.equal(done.result!.securitySummary, '自动复审失败:审计 CC 崩了');
const runs = store.listRuns(t.id);
assert.equal(runs.find((r) => r.kind === 'executor')!.status, 'succeeded');
assert.equal(runs.find((r) => r.kind === 'reviewer')!.status, 'succeeded');
const security = runs.find((r) => r.kind === 'security')!;
assert.equal(security.status, 'failed');
assert.match(security.error ?? '', /审计 CC 崩了/);
store.close();
});
test('verify 不过:按失败处理(run failed + 重新入队)', async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'v', complexity: 'easy' });
const orch = createOrchestrator(store, noopLog, mockDeps({
verify: async () => ({ ok: false, exitCode: 1, logRef: '/tmp/v.log', error: 'verify 失败(exit 1' }),
}));
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued'); // 第一次失败 → 重新入队
const runs = store.listRuns(t.id);
assert.equal(runs.length, 1);
assert.equal(runs[0].status, 'failed');
assert.match(runs[0].error ?? '', /verify 失败/);
store.close();
});
test(`失败重试:重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention(共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run`, async () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });
let attempts = 0;
// 使用快进时钟跳过退避冷却(每次 tick 前将时钟推进 10min
const clock = makeClock();
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
nowMs: clock.nowMs,
}));
for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
clock.advance(10 * 60_000); // 跳过退避冷却
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued', `${i} 次失败后应重新入队`);
}
clock.advance(10 * 60_000); // 跳过最后一次退避
orch.tick(); // 最后一次重试也失败
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);
clock.advance(10 * 60_000);
orch.tick(); // needs_attention 不会再被领取
await orch.drain();
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
store.close();
});
test('项目级 maxRetries=1:只重试 1 次就 → needs_attention', async () => {
const store = new Store(':memory:');
const p = store.createProject({
name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
autonomy: 'auto-easy', maxRetries: 1,
});
const projectId = p.id;
const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });
let attempts = 0;
const clock = makeClock();
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
nowMs: clock.nowMs,
}));
// 第 1 次执行失败 → 重新入队(还有 1 次重试机会)
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued');
// 第 2 次失败 → 超过 maxRetries=1 → needs_attention(快进时钟跳过退避)
clock.advance(10 * 60_000);
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, 2);
store.close();
});
test('项目级 maxRetries=0:首次失败直接 needs_attention(不重试)', async () => {
const store = new Store(':memory:');
const p = store.createProject({
name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
autonomy: 'auto-easy', maxRetries: 0,
});
const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });
let attempts = 0;
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
}));
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, 1);
store.close();
});
test('computeBackoffMs:指数退避公式,30s 基数,上限 10min', () => {
assert.equal(computeBackoffMs(1), 30_000);
assert.equal(computeBackoffMs(2), 60_000);
assert.equal(computeBackoffMs(3), 120_000);
assert.equal(computeBackoffMs(4), 240_000);
assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
assert.equal(computeBackoffMs(100), 10 * 60_000);
});
test('退避冷却:首次失败重入队后,退避期内不被领取;退避过期后正常领取', async () => {
test('退避:nextEligibleAt 在未来 → 不领;过期后 → 领', () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
// 任务已在 queued 且退避到未来
store.transition(t.id, 'queued', { by: 'test' });
const future = new Date(Date.now() + 60_000).toISOString();
store.setNextEligibleAt(t.id, future);
let attempts = 0;
// 使用可控时钟:初始时间 0,退避 30sattempt=1 → BACKOFF_BASE=30000ms
const clock = makeClock(0);
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
nowMs: clock.nowMs,
}));
const clock = makeClock(Date.now());
const state = freshState();
const { deps } = mockDeps(state, { nowMs: clock.nowMs });
const orch = createOrchestrator(store, noopLog, deps);
// 第 1 次失败(t=0)→ 重入队 + 退避(backoffUntil = 0 + 30000 = 30000ms
orch.tick();
await orch.drain();
assert.equal(store.getTask(t.id)!.status, 'queued');
assert.equal(attempts, 1);
// t=29999:退避期内,不被领取
clock.advance(29_999);
orch.tick();
await orch.drain();
assert.equal(attempts, 1, '退避期内不应再次执行');
orch.claimTick();
assert.equal(state.spawned.length, 0, '退避期内不领');
assert.equal(store.getTask(t.id)!.status, 'queued');
// t=30001:退避过期,任务应被重新领取
clock.advance(2);
orch.tick();
await orch.drain();
assert.equal(attempts, 2, '退避过期后应再次执行');
clock.advance(61_000); // 退避过期
orch.claimTick();
assert.equal(state.spawned.length, 1, '退避过期后领取');
assert.equal(store.getTask(t.id)!.status, 'executing');
store.close();
});
test('requeueTaskneeds_attention → queued,重置重试基线,再次允许重试', async () => {
test('spawnWorker 抛错:兜底 failTaskAttempttask 转 queuedexecutor run failed', () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });
const t = store.createTask({ projectId, title: 'boom-spawn', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state, {
spawnWorker: () => { throw new Error('spawn 失败:ENOENT'); },
});
createOrchestrator(store, noopLog, deps).claimTick();
let attempts = 0;
const clock = makeClock();
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
nowMs: clock.nowMs,
}));
const after = store.getTask(t.id)!;
assert.equal(after.status, 'queued', '默认 maxRetries=2,首次失败 → 重入队');
assert.ok(after.nextEligibleAt, '退避已持久化');
const executor = store.listRuns(t.id).find((r) => r.kind === 'executor')!;
assert.equal(executor.status, 'failed');
assert.match(executor.error ?? '', /spawn 失败/);
store.close();
});
// 耗尽默认重试次数 → needs_attention(每次需快进时钟跳过退避)
for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
clock.advance(10 * 60_000);
orch.tick();
await orch.drain();
}
// ───────────────────────── reaper(回收死 worker)─────────────────────────
test('reaperisWorkerAlive=false → failTaskAttemptexecuting→queuedexecutor run failed', () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'dead', complexity: 'easy' });
// 先让它进 executing + 建 run(用一次正常领取,alive=true 不会被回收)
const state = freshState();
const { deps } = mockDeps(state);
const orch = createOrchestrator(store, noopLog, deps);
orch.claimTick();
assert.equal(store.getTask(t.id)!.status, 'executing');
const runId = store.listRuns(t.id)[0].id;
// 标记 worker 已死 → reaper 回收
state.alive = false;
orch.reap();
const after = store.getTask(t.id)!;
assert.equal(after.status, 'queued', '默认 maxRetries=2,首次回收 → 重入队');
assert.ok(after.nextEligibleAt, '退避已持久化');
const executor = store.getRun(runId)!;
assert.equal(executor.status, 'failed');
assert.match(executor.error ?? '', /worker 异常退出/);
store.close();
});
test('reaperisWorkerAlive=true → 不回收(保持 executing', () => {
const { store, projectId } = setup('auto-easy');
const t = store.createTask({ projectId, title: 'alive', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state);
const orch = createOrchestrator(store, noopLog, deps);
orch.claimTick();
assert.equal(store.getTask(t.id)!.status, 'executing');
state.alive = true;
orch.reap();
assert.equal(store.getTask(t.id)!.status, 'executing', 'worker 活 → 不回收');
store.close();
});
test('reapermaxRetries=0 → 死 worker 直接 needs_attention', () => {
const store = new Store(':memory:');
const p = store.createProject({
name: 'orch', repoPath: '/tmp/orch-reap0-' + Math.random(), autonomy: 'auto-easy', maxRetries: 0,
});
const t = store.createTask({ projectId: p.id, title: 'x', complexity: 'easy' });
const state = freshState();
const { deps } = mockDeps(state);
const orch = createOrchestrator(store, noopLog, deps);
orch.claimTick();
state.alive = false;
orch.reap();
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
store.close();
});
// 手动重投:重置基线,转 queued
const requeued = store.requeueTask(t.id);
assert.equal(requeued.status, 'queued');
assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);
// ───────────────────────── 多轮:reaper 腾槽后重领(退避到期)─────────────────────────
// 重投后编排器应能再次执行(快进时钟确保无退避阻拦)
test('死 worker 回收 + 退避到期后下一轮重新领取(监工闭环)', () => {
const { store, projectId } = setup('auto-easy', 1);
const t = store.createTask({ projectId, title: 'recycle', complexity: 'easy' });
const clock = makeClock(Date.now());
const state = freshState();
const { deps } = mockDeps(state, { nowMs: clock.nowMs });
const orch = createOrchestrator(store, noopLog, deps);
// 第一轮:领取 + spawnalive=true
orch.tick();
assert.equal(state.spawned.length, 1);
assert.equal(store.getTask(t.id)!.status, 'executing');
// worker 死 → 下一轮 reaper 回收(→ queued + 退避),退避期内不重领
state.alive = false;
orch.tick();
assert.equal(store.getTask(t.id)!.status, 'queued');
assert.equal(state.spawned.length, 1, '退避期内不重领');
// 退避到期 → 再下一轮重新领取
clock.advance(10 * 60_000);
orch.tick();
await orch.drain();
assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
// 第 1 次净失败后应重入队(基线已重置,净失败=1 < maxRetries=2
assert.equal(store.getTask(t.id)!.status, 'queued');
assert.equal(state.spawned.length, 2, '退避到期 → 重新领取 spawn');
assert.equal(store.getTask(t.id)!.status, 'executing');
store.close();
});
test('project paused:不领取', async () => {
const { store, projectId } = setup('auto-approved');
const t = store.createTask({ projectId, title: 'x', complexity: 'easy' });
store.patchProject(projectId, { status: 'paused' });
let calls = 0;
const orch = createOrchestrator(store, noopLog, mockDeps({
runner: async () => { calls++; return okRun; },
}));
orch.tick();
await orch.drain();
assert.equal(calls, 0);
assert.equal(store.getTask(t.id)!.status, 'ready');
test(`reaper 反复回收:净失败 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention`, () => {
const { store, projectId } = setup('auto-easy', 1);
const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });
const clock = makeClock(Date.now());
const state = freshState();
const { deps } = mockDeps(state, { nowMs: clock.nowMs });
const orch = createOrchestrator(store, noopLog, deps);
// 反复:领取(alive=true 领取那刻)→ 标死 → reaper 回收
for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
clock.advance(10 * 60_000); // 跳过退避
state.alive = true;
orch.claimTick();
state.alive = false;
orch.reap();
}
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
const failed = store.listRuns(t.id).filter((r) => r.kind === 'executor' && r.status === 'failed');
assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);
store.close();
});