maestro/test/orchestrator.test.ts

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { Store } from '../src/store/index.js';
import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
import type { RunnerResult } from '../src/executor/runner.js';
import type { ReviewResult } from '../src/executor/reviewer.js';
import type { Autonomy } from '../src/model/types.js';

const noopLog = { info: (): void => undefined, error: (): void => undefined };

function deferred<T>(): { promise: Promise<T>; resolve: (v: T) => void } {
  let resolve!: (v: T) => void;
  const promise = new Promise<T>((r) => { resolve = r; });
  return { promise, resolve };
}

const okRun: RunnerResult = { ok: true, transcriptRef: '/tmp/fake.jsonl', sessionId: 'sess-mock-1', finalText: '执行自述：改了 README' };

const okReview: ReviewResult = {
  summary: '## 做了什么\nmock 复审通过',
  verdict: 'approve',
  transcriptRef: '/tmp/fake-review.jsonl',
  sessionId: 'sess-review-1',
};

const okSecurity: ReviewResult = {
  summary: '## 安全审计\nmock 审计通过',
  verdict: 'approve',
  transcriptRef: '/tmp/fake-security.jsonl',
  sessionId: 'sess-security-1',
};

/** 全 mock 依赖（不真起 CC、不动 git）：可按用例覆盖 */
function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
  return {
    createWorktree: async (_repo, taskId) => ({ dir: `/tmp/fake-wt/${taskId}`, branch: `maestro/${taskId}` }),
    worktreeDiff: async () => ({ diffSummary: ' README.md | 1 +', commits: ['abc1234 hello maestro'] }),
    verify: async () => ({ ok: true, exitCode: 0, logRef: null }),
    runner: async () => okRun,
    reviewCode: async () => okReview,
    reviewSecurity: async () => okSecurity,
    nowMs: Date.now,
    ...overrides,
  };
}

/** 可前进的测试时钟（用于退避相关测试） */
function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
  let t = initialMs;
  return { nowMs: () => t, advance: (ms) => { t += ms; } };
}

function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
  const store = new Store(':memory:');
  const p = store.createProject({
    name: 'orch', repoPath: '/tmp/orch-repo-' + Math.random(), autonomy, concurrency,
  });
  return { store, projectId: p.id };
}

async function settle(): Promise<void> {
  await new Promise((r) => setImmediate(r));
}

test('autonomy=manual：编排器不领取任何任务', async () => {
  const { store, projectId } = setup('manual');
  const t = store.createTask({ projectId, title: 'easy task', complexity: 'easy' });
  let calls = 0;
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { calls++; return okRun; },
  }));
  orch.tick();
  await orch.drain();
  assert.equal(calls, 0);
  assert.equal(store.getTask(t.id)!.status, 'ready');
  store.close();
});

test('autonomy=auto-easy：只领 easy，medium ready 不动', async () => {
  const { store, projectId } = setup('auto-easy', 5);
  const easy = store.createTask({ projectId, title: 'small', complexity: 'easy' });
  const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
  store.setSpec(medium.id, '方案');
  store.transition(medium.id, 'spec_review');
  store.decide(medium.id, 'accept', 'user');                 // medium → ready
  assert.equal(store.getTask(medium.id)!.status, 'ready');

  const ran: string[] = [];
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async (task) => { ran.push(task.id); return okRun; },
  }));
  orch.tick();
  await orch.drain();

  assert.deepEqual(ran, [easy.id]);
  assert.equal(store.getTask(easy.id)!.status, 'exec_review');
  assert.equal(store.getTask(medium.id)!.status, 'ready');   // 不自动跑
  store.close();
});

test('autonomy=auto-approved：领全部 ready（含 medium），依赖未满足/非叶子不领', async () => {
  const { store, projectId } = setup('auto-approved', 5);
  const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
  store.setSpec(medium.id, '方案');
  store.transition(medium.id, 'spec_review');
  store.decide(medium.id, 'accept', 'user');
  // 依赖未 done 的任务：建在 medium 上 → blocked，不可领
  const dep = store.createTask({ projectId, title: 'after-mid', complexity: 'easy', deps: [medium.id] });
  assert.equal(store.getTask(dep.id)!.status, 'blocked');

  const ran: string[] = [];
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async (task) => { ran.push(task.id); return okRun; },
  }));
  orch.tick();
  await orch.drain();

  assert.deepEqual(ran, [medium.id]);
  assert.equal(store.getTask(medium.id)!.status, 'exec_review');
  assert.equal(store.getTask(dep.id)!.status, 'blocked');
  store.close();
});

test('concurrency=1：同项目同轮只领 1 个，跑完下一轮再领', async () => {
  const { store, projectId } = setup('auto-approved', 1);
  const t1 = store.createTask({ projectId, title: 'a', complexity: 'easy' });
  const t2 = store.createTask({ projectId, title: 'b', complexity: 'easy' });

  const gate = deferred<void>();
  const started: string[] = [];
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async (task) => { started.push(task.id); await gate.promise; return okRun; },
  }));

  orch.tick();
  await settle();
  assert.deepEqual(started, [t1.id], '并发=1 只应启动第一个任务');
  assert.equal(store.getTask(t1.id)!.status, 'executing');
  assert.equal(store.getTask(t2.id)!.status, 'ready');
  assert.equal(orch.inflight.size, 1);

  orch.tick();                              // 在途占满 → 本轮不领
  await settle();
  assert.deepEqual(started, [t1.id]);

  gate.resolve();
  await orch.drain();
  assert.equal(store.getTask(t1.id)!.status, 'exec_review');

  orch.tick();                              // 槽位释放 → 领第二个
  gate.resolve();
  await orch.drain();
  assert.deepEqual(started, [t1.id, t2.id]);
  assert.equal(store.getTask(t2.id)!.status, 'exec_review');
  store.close();
});

test('成功路径：状态流转 + setResult(四字段) + executor/reviewer/security 三 run succeeded', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'tweak', complexity: 'easy' });
  store.setOperations(t.id, '在 README.md 追加一行');

  const seen: string[] = [];
  store.subscribe((e) => { if (e.type === 'status.changed' && e.taskId === t.id) seen.push(String(e.payload.to)); });

  let codeReportSeen: string | null = null;
  let secReportSeen: string | null = null;
  const orch = createOrchestrator(store, noopLog, mockDeps({
    reviewCode: async (_task, _project, _wt, _runId, executorReport) => {
      codeReportSeen = executorReport;
      return okReview;
    },
    reviewSecurity: async (_task, _project, _wt, _runId, executorReport) => {
      secReportSeen = executorReport;
      return okSecurity;
    },
  }));
  orch.tick();
  await orch.drain();

  const done = store.getTask(t.id)!;
  assert.equal(done.status, 'exec_review');
  assert.deepEqual(seen, ['queued', 'executing', 'exec_review']);
  assert.deepEqual(done.result, {
    branch: `maestro/${t.id}`,
    worktree: `/tmp/fake-wt/${t.id}`,
    diffSummary: ' README.md | 1 +',
    commits: ['abc1234 hello maestro'],
    prUrl: null,
    summary: '## 做了什么\nmock 复审通过',
    verdict: 'approve',
    securitySummary: '## 安全审计\nmock 审计通过',
    securityVerdict: 'approve',
    mergeTaskId: null,
  });
  assert.equal(codeReportSeen, '执行自述：改了 README');   // runner finalText 传给两个复审作执行者自述
  assert.equal(secReportSeen, '执行自述：改了 README');

  const runs = store.listRuns(t.id);
  assert.equal(runs.length, 3);
  const executor = runs.find((r) => r.kind === 'executor')!;
  assert.equal(executor.status, 'succeeded');
  assert.equal(executor.branch, `maestro/${t.id}`);
  assert.equal(executor.transcriptRef, '/tmp/fake.jsonl');
  assert.equal(executor.claudeSessionId, 'sess-mock-1');
  const reviewer = runs.find((r) => r.kind === 'reviewer')!;
  assert.equal(reviewer.status, 'succeeded');
  assert.equal(reviewer.transcriptRef, '/tmp/fake-review.jsonl');
  assert.equal(reviewer.claudeSessionId, 'sess-review-1');
  const security = runs.find((r) => r.kind === 'security')!;
  assert.equal(security.status, 'succeeded');
  assert.equal(security.transcriptRef, '/tmp/fake-security.jsonl');
  assert.equal(security.claudeSessionId, 'sess-security-1');
  store.close();
});

test('任一 verdict=reject 也照常落进 result（最终裁决仍归用户）', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'risky', complexity: 'easy' });
  const orch = createOrchestrator(store, noopLog, mockDeps({
    reviewCode: async () => ({ ...okReview, summary: '发现问题', verdict: 'reject' as const }),
    reviewSecurity: async () => ({ ...okSecurity, summary: '发现密钥泄露', verdict: 'reject' as const }),
  }));
  orch.tick();
  await orch.drain();
  const done = store.getTask(t.id)!;
  assert.equal(done.status, 'exec_review');
  assert.equal(done.result!.verdict, 'reject');
  assert.equal(done.result!.summary, '发现问题');
  assert.equal(done.result!.securityVerdict, 'reject');
  assert.equal(done.result!.securitySummary, '发现密钥泄露');
  store.close();
});

test('code review 失败不挡任务：summary 记失败原因、verdict=null，安全审计照常跑', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'review-broken', complexity: 'easy' });
  const orch = createOrchestrator(store, noopLog, mockDeps({
    reviewCode: async () => { throw new Error('复审 CC 崩了'); },
  }));
  orch.tick();
  await orch.drain();

  const done = store.getTask(t.id)!;
  assert.equal(done.status, 'exec_review');                       // 不挡结果闸
  assert.equal(done.result!.verdict, null);
  assert.equal(done.result!.summary, '自动复审失败：复审 CC 崩了');
  assert.equal(done.result!.securityVerdict, 'approve');          // 另一个复审不受影响
  assert.equal(done.result!.securitySummary, '## 安全审计\nmock 审计通过');

  const runs = store.listRuns(t.id);
  assert.equal(runs.find((r) => r.kind === 'executor')!.status, 'succeeded');
  const reviewer = runs.find((r) => r.kind === 'reviewer')!;
  assert.equal(reviewer.status, 'failed');
  assert.match(reviewer.error ?? '', /复审 CC 崩了/);
  assert.equal(runs.find((r) => r.kind === 'security')!.status, 'succeeded');
  store.close();
});

test('安全审计失败不挡任务：securitySummary 记失败原因、securityVerdict=null，code review 不受影响', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'security-broken', complexity: 'easy' });
  const orch = createOrchestrator(store, noopLog, mockDeps({
    reviewSecurity: async () => { throw new Error('审计 CC 崩了'); },
  }));
  orch.tick();
  await orch.drain();

  const done = store.getTask(t.id)!;
  assert.equal(done.status, 'exec_review');
  assert.equal(done.result!.verdict, 'approve');
  assert.equal(done.result!.summary, '## 做了什么\nmock 复审通过');
  assert.equal(done.result!.securityVerdict, null);
  assert.equal(done.result!.securitySummary, '自动复审失败：审计 CC 崩了');

  const runs = store.listRuns(t.id);
  assert.equal(runs.find((r) => r.kind === 'executor')!.status, 'succeeded');
  assert.equal(runs.find((r) => r.kind === 'reviewer')!.status, 'succeeded');
  const security = runs.find((r) => r.kind === 'security')!;
  assert.equal(security.status, 'failed');
  assert.match(security.error ?? '', /审计 CC 崩了/);
  store.close();
});

test('verify 不过：按失败处理（run failed + 重新入队）', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'v', complexity: 'easy' });
  const orch = createOrchestrator(store, noopLog, mockDeps({
    verify: async () => ({ ok: false, exitCode: 1, logRef: '/tmp/v.log', error: 'verify 失败（exit 1）' }),
  }));
  orch.tick();
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'queued');       // 第一次失败 → 重新入队
  const runs = store.listRuns(t.id);
  assert.equal(runs.length, 1);
  assert.equal(runs[0].status, 'failed');
  assert.match(runs[0].error ?? '', /verify 失败/);
  store.close();
});

test(`失败重试：重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention（共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run）`, async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });

  let attempts = 0;
  // 使用快进时钟跳过退避冷却（每次 tick 前将时钟推进 10min）
  const clock = makeClock();
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
    nowMs: clock.nowMs,
  }));

  for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
    clock.advance(10 * 60_000);            // 跳过退避冷却
    orch.tick();
    await orch.drain();
    assert.equal(store.getTask(t.id)!.status, 'queued', `第 ${i} 次失败后应重新入队`);
  }

  clock.advance(10 * 60_000);             // 跳过最后一次退避
  orch.tick();                              // 最后一次重试也失败
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);

  const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
  assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);

  clock.advance(10 * 60_000);
  orch.tick();                              // needs_attention 不会再被领取
  await orch.drain();
  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
  store.close();
});

test('项目级 maxRetries=1：只重试 1 次就 → needs_attention', async () => {
  const store = new Store(':memory:');
  const p = store.createProject({
    name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
    autonomy: 'auto-easy', maxRetries: 1,
  });
  const projectId = p.id;
  const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });

  let attempts = 0;
  const clock = makeClock();
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
    nowMs: clock.nowMs,
  }));

  // 第 1 次执行失败 → 重新入队（还有 1 次重试机会）
  orch.tick();
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'queued');

  // 第 2 次失败 → 超过 maxRetries=1 → needs_attention（快进时钟跳过退避）
  clock.advance(10 * 60_000);
  orch.tick();
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
  assert.equal(attempts, 2);
  store.close();
});

test('项目级 maxRetries=0：首次失败直接 needs_attention（不重试）', async () => {
  const store = new Store(':memory:');
  const p = store.createProject({
    name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
    autonomy: 'auto-easy', maxRetries: 0,
  });
  const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });

  let attempts = 0;
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
  }));

  orch.tick();
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
  assert.equal(attempts, 1);
  store.close();
});

test('computeBackoffMs：指数退避公式，30s 基数，上限 10min', () => {
  assert.equal(computeBackoffMs(1), 30_000);
  assert.equal(computeBackoffMs(2), 60_000);
  assert.equal(computeBackoffMs(3), 120_000);
  assert.equal(computeBackoffMs(4), 240_000);
  assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
  assert.equal(computeBackoffMs(100), 10 * 60_000);
});

test('退避冷却：首次失败重入队后，退避期内不被领取；退避过期后正常领取', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });

  let attempts = 0;
  // 使用可控时钟：初始时间 0，退避 30s（attempt=1 → BACKOFF_BASE=30000ms）
  const clock = makeClock(0);
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
    nowMs: clock.nowMs,
  }));

  // 第 1 次失败（t=0）→ 重入队 + 退避（backoffUntil = 0 + 30000 = 30000ms）
  orch.tick();
  await orch.drain();
  assert.equal(store.getTask(t.id)!.status, 'queued');
  assert.equal(attempts, 1);

  // t=29999：退避期内，不被领取
  clock.advance(29_999);
  orch.tick();
  await orch.drain();
  assert.equal(attempts, 1, '退避期内不应再次执行');
  assert.equal(store.getTask(t.id)!.status, 'queued');

  // t=30001：退避过期，任务应被重新领取
  clock.advance(2);
  orch.tick();
  await orch.drain();
  assert.equal(attempts, 2, '退避过期后应再次执行');
  store.close();
});

test('requeueTask：needs_attention → queued，重置重试基线，再次允许重试', async () => {
  const { store, projectId } = setup('auto-easy');
  const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });

  let attempts = 0;
  const clock = makeClock();
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
    nowMs: clock.nowMs,
  }));

  // 耗尽默认重试次数 → needs_attention（每次需快进时钟跳过退避）
  for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
    clock.advance(10 * 60_000);
    orch.tick();
    await orch.drain();
  }
  assert.equal(store.getTask(t.id)!.status, 'needs_attention');
  assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);

  // 手动重投：重置基线，转 queued
  const requeued = store.requeueTask(t.id);
  assert.equal(requeued.status, 'queued');
  assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);

  // 重投后编排器应能再次执行（快进时钟确保无退避阻拦）
  clock.advance(10 * 60_000);
  orch.tick();
  await orch.drain();
  assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
  // 第 1 次净失败后应重入队（基线已重置，净失败=1 < maxRetries=2）
  assert.equal(store.getTask(t.id)!.status, 'queued');
  store.close();
});

test('project paused：不领取', async () => {
  const { store, projectId } = setup('auto-approved');
  const t = store.createTask({ projectId, title: 'x', complexity: 'easy' });
  store.patchProject(projectId, { status: 'paused' });
  let calls = 0;
  const orch = createOrchestrator(store, noopLog, mockDeps({
    runner: async () => { calls++; return okRun; },
  }));
  orch.tick();
  await orch.drain();
  assert.equal(calls, 0);
  assert.equal(store.getTask(t.id)!.status, 'ready');
  store.close();
});