phase3(daemon): 监工 tick(spawn worker)+ ingest(outbox→DB)+ reaper + reconcile 接真判活
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+10
-1
@@ -6,6 +6,7 @@ import { registerStatic } from '../api/static.js';
|
|||||||
import { loadConfig } from './config.js';
|
import { loadConfig } from './config.js';
|
||||||
import { startOrchestrator } from './orchestrator.js';
|
import { startOrchestrator } from './orchestrator.js';
|
||||||
import { startNotifier } from './notify.js';
|
import { startNotifier } from './notify.js';
|
||||||
|
import { isWorkerAlive, heartbeatAgeMs } from '../executor/protocol.js';
|
||||||
import { syncProject, todoJsonPath } from '../sync/todo-sync.js';
|
import { syncProject, todoJsonPath } from '../sync/todo-sync.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -53,7 +54,15 @@ async function main(): Promise<void> {
|
|||||||
const cfg = loadConfig();
|
const cfg = loadConfig();
|
||||||
const store = new Store(cfg.dbFile);
|
const store = new Store(cfg.dbFile);
|
||||||
const rec = store.reconcileDeps(); // 启动对账:ready↔blocked 按依赖纠正存量数据
|
const rec = store.reconcileDeps(); // 启动对账:ready↔blocked 按依赖纠正存量数据
|
||||||
const itr = store.reconcileInterrupted(); // 中断恢复:上次退出时在跑的任务重新入队
|
// 中断恢复(多进程执行):用 worker_pid + 心跳真判活——worker 仍活则 re-adopt(daemon 续 ingest 其 outbox),
|
||||||
|
// 死则回收(failTaskAttempt 收尾 + 重试/needs_attention)。
|
||||||
|
const itr = store.reconcileInterrupted((run) =>
|
||||||
|
run !== null && isWorkerAlive({
|
||||||
|
pid: run.workerPid,
|
||||||
|
heartbeatAgeMs: heartbeatAgeMs(run.id),
|
||||||
|
startedAgeMs: Date.now() - Date.parse(run.startedAt),
|
||||||
|
}),
|
||||||
|
);
|
||||||
const app = buildServer({ store, logger: true });
|
const app = buildServer({ store, logger: true });
|
||||||
|
|
||||||
registerStatic(app); // Web 看板(web/ 静态文件)
|
registerStatic(app); // Web 看板(web/ 静态文件)
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
import { Store } from '../store/index.js';
|
||||||
|
import { readOutboxSince, type OutboxRecord } from '../executor/protocol.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Outbox → DB 摄取(daemon 是唯一 DB 写者)。
|
||||||
|
*
|
||||||
|
* worker 全程不碰 DB,只把进度/结果追加进 runs/<runId>/outbox.ndjson。daemon 在每轮 tick 里把
|
||||||
|
* 这些记录读出来,按 seq 升序逐条映射成 DB 写(finishRun / setResult / transition / failTaskAttempt …)。
|
||||||
|
*
|
||||||
|
* 幂等靠 run.lastSeq 游标:只处理 seq>lastSeq 的记录,每处理一条就 setLastSeq(seq)。崩溃重启续读时
|
||||||
|
* 已落库的 seq 不会重做——故 result/failed/done 这类终态记录即便文件还在,也只会被消费一次。
|
||||||
|
*/
|
||||||
|
export interface IngestLogger {
|
||||||
|
info(msg: string): void;
|
||||||
|
error(msg: string): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 摄取单个 run 的新 outbox 记录(seq>run.lastSeq),按序落库。
|
||||||
|
* run 不存在则忽略(已被删/异常)。每条处理后推进 lastSeq 游标。
|
||||||
|
*/
|
||||||
|
export function ingestRun(store: Store, log: IngestLogger, runId: string): void {
|
||||||
|
const run = store.getRun(runId);
|
||||||
|
if (!run) return;
|
||||||
|
const taskId = run.taskId;
|
||||||
|
const recs = readOutboxSince(runId, run.lastSeq);
|
||||||
|
for (const rec of recs) {
|
||||||
|
try {
|
||||||
|
applyRecord(store, log, taskId, runId, rec);
|
||||||
|
} catch (e) {
|
||||||
|
// 单条映射失败不阻断后续 run 的摄取;但本条不推进游标,下轮重试。
|
||||||
|
log.error(`ingest run=${runId} seq=${rec.seq} 失败:${(e as Error).message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
store.setLastSeq(runId, rec.seq);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 把一条 OutboxRecord 映射为 DB 写。 */
|
||||||
|
function applyRecord(store: Store, log: IngestLogger, taskId: string, runId: string, rec: OutboxRecord): void {
|
||||||
|
switch (rec.type) {
|
||||||
|
case 'started':
|
||||||
|
// worker 启动自报;pid 已由 daemon spawn 时 setWorkerPid 写过,这里仅记日志。
|
||||||
|
log.info(`worker 启动 task=${taskId} run=${runId} pid=${rec.pid} model=${rec.model}`);
|
||||||
|
return;
|
||||||
|
|
||||||
|
case 'phase':
|
||||||
|
// 进度阶段,仅日志(看板事件可后续接)。
|
||||||
|
log.info(`task=${taskId} run=${runId} 阶段=${rec.phase}`);
|
||||||
|
return;
|
||||||
|
|
||||||
|
case 'failed': {
|
||||||
|
// 先把 executor run 收尾为 failed(带转录/会话),再走失败/重试策略。
|
||||||
|
// failTaskAttempt 见 run 已 ended(非 started)不会重复收尾,只做重试决策。
|
||||||
|
try {
|
||||||
|
store.finishRun(runId, 'failed', {
|
||||||
|
error: rec.error,
|
||||||
|
transcriptRef: rec.transcriptRef ?? undefined,
|
||||||
|
claudeSessionId: rec.sessionId ?? undefined,
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
log.error(`task=${taskId} run=${runId} 收尾 failed 出错(继续重试决策):${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
store.failTaskAttempt(taskId, runId, rec.error);
|
||||||
|
log.info(`task=${taskId} run=${runId} 执行失败:${rec.error}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'result': {
|
||||||
|
// 成功终态:建 reviewer/security 两条复审 run + 收尾,写 result 四字段,转 exec_review,收尾 executor run。
|
||||||
|
const cr = store.startRun(taskId, 'reviewer', { worktree: rec.worktree, branch: rec.branch });
|
||||||
|
store.finishRun(cr.id, 'succeeded', { transcriptRef: rec.code.transcriptRef ?? undefined });
|
||||||
|
|
||||||
|
const sr = store.startRun(taskId, 'security', { worktree: rec.worktree, branch: rec.branch });
|
||||||
|
store.finishRun(sr.id, 'succeeded', { transcriptRef: rec.security.transcriptRef ?? undefined });
|
||||||
|
|
||||||
|
store.setResult(taskId, {
|
||||||
|
branch: rec.branch,
|
||||||
|
worktree: rec.worktree,
|
||||||
|
diffSummary: rec.diffSummary,
|
||||||
|
commits: rec.commits,
|
||||||
|
prUrl: null,
|
||||||
|
summary: rec.code.summary,
|
||||||
|
verdict: rec.code.verdict,
|
||||||
|
securitySummary: rec.security.summary,
|
||||||
|
securityVerdict: rec.security.verdict,
|
||||||
|
mergeTaskId: null,
|
||||||
|
});
|
||||||
|
store.transition(taskId, 'exec_review', { by: 'ingest', runId });
|
||||||
|
store.finishRun(runId, 'succeeded', {
|
||||||
|
transcriptRef: rec.executor.transcriptRef ?? undefined,
|
||||||
|
claudeSessionId: rec.executor.sessionId ?? undefined,
|
||||||
|
});
|
||||||
|
log.info(`task=${taskId} run=${runId} 执行完成 → exec_review(${rec.commits.length} commits)`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'done':
|
||||||
|
// worker 即将退出:终态标记。run 已由 result/failed 收尾,这里无须额外 DB 写。
|
||||||
|
log.info(`task=${taskId} run=${runId} worker 退出`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 摄取所有在途 run:对每个 executing 任务的最近一条 executor run 调 ingestRun。
|
||||||
|
* daemon 每轮 tick 调一次,把 worker 期间累积的 outbox 落库。
|
||||||
|
*/
|
||||||
|
export function ingestAll(store: Store, log: IngestLogger): void {
|
||||||
|
for (const { run } of store.executingWithLatestExecutorRun()) {
|
||||||
|
if (!run) continue;
|
||||||
|
ingestRun(store, log, run.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
+144
-168
@@ -1,66 +1,107 @@
|
|||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
import { Store } from '../store/index.js';
|
import { Store } from '../store/index.js';
|
||||||
import type { Project, Task, ReviewVerdict } from '../model/types.js';
|
import type { Project, Task } from '../model/types.js';
|
||||||
import { rankByScore } from '../model/scoring.js';
|
import { rankByScore } from '../model/scoring.js';
|
||||||
import { createWorktree, worktreeDiff, type WorktreeDiff, type WorktreeInfo } from '../executor/worktree.js';
|
import { branchFor, worktreeDirFor } from '../executor/worktree.js';
|
||||||
import { runTask, type RunnerFn } from '../executor/runner.js';
|
import { writeJobSpec, isWorkerAlive, heartbeatAgeMs, type JobSpec } from '../executor/protocol.js';
|
||||||
import { runVerify, type VerifyFn } from '../executor/verify.js';
|
|
||||||
import { reviewCode, reviewSecurity, type ReviewerFn } from '../executor/reviewer.js';
|
|
||||||
import { pickModel } from '../executor/models.js';
|
import { pickModel } from '../executor/models.js';
|
||||||
|
import { ingestAll, type IngestLogger } from './ingest.js';
|
||||||
|
|
||||||
/** 失败后默认最多自动重试次数(项目级 maxRetries 未设置时回退此值) */
|
/** 失败后默认最多自动重试次数(项目级 maxRetries 未设置时回退此值)。失败/退避策略本体在 store.failTaskAttempt。 */
|
||||||
export const DEFAULT_MAX_RETRIES = 2;
|
export const DEFAULT_MAX_RETRIES = 2;
|
||||||
|
|
||||||
/** 指数退避基础等待(第 n 次重试等待 BASE * 2^(n-1),上限 MAX) */
|
|
||||||
const BACKOFF_BASE_MS = 30_000; // 30s
|
|
||||||
const BACKOFF_MAX_MS = 10 * 60_000; // 10min
|
|
||||||
|
|
||||||
/** 第 attempt 次重试(1-indexed)的等待毫秒 */
|
|
||||||
export function computeBackoffMs(attempt: number): number {
|
|
||||||
return Math.min(BACKOFF_BASE_MS * (2 ** (attempt - 1)), BACKOFF_MAX_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OrchestratorLogger {
|
export interface OrchestratorLogger {
|
||||||
info(msg: string): void;
|
info(msg: string): void;
|
||||||
error(msg: string): void;
|
error(msg: string): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 依赖注入点:测试传 mock,生产用真实现 */
|
/**
|
||||||
|
* worker 入口解析:返回 spawn 用的 [cmd, ...args](不含 runId,由调用方追加)。
|
||||||
|
* - 默认:node <本文件同级 ../executor/worker.js>(编译后 dist 布局:daemon/ 与 executor/ 同级)。
|
||||||
|
* - env MAESTRO_WORKER_CMD 覆盖整条命令(空格分隔),供测试 / tsx 跑 .ts 入口用,
|
||||||
|
* 例:MAESTRO_WORKER_CMD="npx tsx src/executor/worker.ts"。
|
||||||
|
*/
|
||||||
|
export function workerEntry(): string[] {
|
||||||
|
const override = process.env.MAESTRO_WORKER_CMD?.trim();
|
||||||
|
if (override) return override.split(/\s+/);
|
||||||
|
const here = dirname(fileURLToPath(import.meta.url));
|
||||||
|
return ['node', join(here, '..', 'executor', 'worker.js')];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* spawn worker 时传给子进程的最小 env:只保留 PATH/HOME/LANG 与 ANTHROPIC_ / CLAUDE_ 前缀变量,
|
||||||
|
* 不把 daemon 全量 env(含 MAESTRO_ 控制变量、无关密钥)漏给独立 worker 进程。
|
||||||
|
* MAESTRO_DATA_DIR 例外透传——worker 必须和 daemon 看同一个 runs/ 目录(job.json/outbox 路径一致)。
|
||||||
|
*/
|
||||||
|
export function workerEnv(src: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
|
||||||
|
const out: NodeJS.ProcessEnv = {};
|
||||||
|
for (const k of ['PATH', 'HOME', 'LANG', 'MAESTRO_DATA_DIR', 'MAESTRO_WORKER_CMD'] as const) {
|
||||||
|
if (src[k] !== undefined) out[k] = src[k];
|
||||||
|
}
|
||||||
|
for (const [k, v] of Object.entries(src)) {
|
||||||
|
if (v !== undefined && (k.startsWith('ANTHROPIC_') || k.startsWith('CLAUDE_'))) out[k] = v;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 默认 spawnWorker:detached + 脱离 stdio + 收敛 env,unref 后自存活(daemon 退出不杀 worker)。返回子进程 pid。 */
|
||||||
|
function defaultSpawnWorker(runId: string): number {
|
||||||
|
const [cmd, ...args] = workerEntry();
|
||||||
|
const child = spawn(cmd, [...args, runId], {
|
||||||
|
detached: true,
|
||||||
|
stdio: 'ignore',
|
||||||
|
env: workerEnv(),
|
||||||
|
});
|
||||||
|
child.unref();
|
||||||
|
if (child.pid === undefined) throw new Error('spawn worker 未返回 pid');
|
||||||
|
return child.pid;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 依赖注入点:生产用默认实现,测试传 mock(不真 spawn / 不真判活 / ingest no-op)。 */
|
||||||
export interface OrchestratorDeps {
|
export interface OrchestratorDeps {
|
||||||
runner: RunnerFn;
|
/** spawn 一个独立 worker 进程跑该 run,返回其 pid。 */
|
||||||
reviewCode: ReviewerFn; // code review(kind=reviewer 的 run)
|
spawnWorker: (runId: string) => number;
|
||||||
reviewSecurity: ReviewerFn; // 安全审计(kind=security 的 run)
|
/** worker 是否存活(reaper 判死用),默认 protocol.isWorkerAlive。 */
|
||||||
verify: VerifyFn;
|
isWorkerAlive: typeof isWorkerAlive;
|
||||||
createWorktree: (repoPath: string, taskId: string, baseBranch: string) => Promise<WorktreeInfo>;
|
/** 把所有在途 run 的 outbox 摄取入库(daemon 唯一写者),默认 ingest.ingestAll。 */
|
||||||
worktreeDiff: (repoPath: string, dir: string, branch: string, baseBranch: string) => Promise<WorktreeDiff>;
|
ingestAll: (store: Store, log: IngestLogger) => void;
|
||||||
/** 当前时间戳(ms);测试注入可控时钟(默认 Date.now) */
|
/** 写 worker 输入 job.json,默认 protocol.writeJobSpec(测试可 mock,避免落盘)。 */
|
||||||
|
writeJobSpec: (job: JobSpec) => void;
|
||||||
|
/** 当前时间戳(ms);测试注入可控时钟(默认 Date.now)。 */
|
||||||
nowMs: () => number;
|
nowMs: () => number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Orchestrator {
|
export interface Orchestrator {
|
||||||
/** 跑一轮领取(同步领取 + 异步执行,不阻塞)。错误只记日志。 */
|
/** 跑一轮:先 ingest(outbox→DB)+ reaper(回收死 worker),再领新任务 spawn worker。错误只记日志、不抛。 */
|
||||||
tick(): void;
|
tick(): void;
|
||||||
/** 等待所有在途执行收尾(测试用) */
|
/** 仅领取/spawn 那一步(测试细粒度断言用,不含 ingest/reaper)。 */
|
||||||
drain(): Promise<void>;
|
claimTick(): void;
|
||||||
/** 在途任务 id(防同任务重复领取) */
|
/** ingest 所有在途 run(测试用)。 */
|
||||||
readonly inflight: ReadonlyMap<string, string>;
|
ingest(): void;
|
||||||
|
/** 回收死 worker(测试用)。 */
|
||||||
|
reap(): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 编排器核心(与定时器解耦,便于测试)。
|
* 编排器(监工):自身不跑 agent,只领任务 + spawn 独立 worker + 把 worker 的 outbox 摄取入库 + 回收死 worker。
|
||||||
* 每轮对 status=active 且 autonomy≠manual 的项目:在途数 < concurrency 时领任务——
|
* 全部 DB 写都在 daemon 进程内(含 ingest 落的 run/result/transition),WS 推送照常经 store.subscribe 触发。
|
||||||
* ready 叶子(deps 全 done;auto-easy 只领 easy)或 queued(重试/重启遗留)。
|
* 无内存在途态——并发闸读 store.countExecuting,退避读 task.nextEligibleAt,故跨 daemon 重启天然持久。
|
||||||
* 成功 → setResult + exec_review;失败 → failed → 重试 ≤MAX_RETRIES 次 → needs_attention。
|
|
||||||
*/
|
*/
|
||||||
export function createOrchestrator(store: Store, log: OrchestratorLogger, deps: Partial<OrchestratorDeps> = {}): Orchestrator {
|
export function createOrchestrator(store: Store, log: OrchestratorLogger, deps: Partial<OrchestratorDeps> = {}): Orchestrator {
|
||||||
const d: OrchestratorDeps = { runner: runTask, reviewCode, reviewSecurity, verify: runVerify, createWorktree, worktreeDiff, nowMs: Date.now, ...deps };
|
const d: OrchestratorDeps = {
|
||||||
const inflight = new Map<string, string>(); // taskId → projectId
|
spawnWorker: defaultSpawnWorker,
|
||||||
const pending = new Set<Promise<void>>();
|
isWorkerAlive,
|
||||||
/** 退避表:taskId → 下次可领取时间戳(ms);daemon 重启后清空,已有 queued 任务即刻可领 */
|
ingestAll,
|
||||||
const backoffUntil = new Map<string, number>();
|
writeJobSpec,
|
||||||
|
nowMs: Date.now,
|
||||||
|
...deps,
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 本项目可领取的任务:queued(重试/孤儿)+ ready 叶子且 deps 全 done;auto-easy 只挑 easy。
|
* 本项目可领取的任务:queued(重试/孤儿)+ ready 叶子且 deps 全 done;auto-easy 只挑 easy。
|
||||||
* 按调度分降序返回(score = 自身分 + 已完成依赖分 + 等待解锁的 blocked 任务分,见 model/scoring.ts)。
|
* 退避读持久化的 task.nextEligibleAt(早于它不领)。按调度分降序返回(见 model/scoring.ts)。
|
||||||
*/
|
*/
|
||||||
function claimable(project: Project): Array<{ task: Task; score: number }> {
|
function claimable(project: Project): Array<{ task: Task; score: number }> {
|
||||||
const nowMs = d.nowMs();
|
const nowMs = d.nowMs();
|
||||||
@@ -69,178 +110,113 @@ export function createOrchestrator(store: Store, log: OrchestratorLogger, deps:
|
|||||||
const parents = new Set(tasks.filter((t) => t.parentId).map((t) => t.parentId as string));
|
const parents = new Set(tasks.filter((t) => t.parentId).map((t) => t.parentId as string));
|
||||||
const easyOnly = project.autonomy === 'auto-easy';
|
const easyOnly = project.autonomy === 'auto-easy';
|
||||||
const candidates = tasks.filter((t) => {
|
const candidates = tasks.filter((t) => {
|
||||||
if (inflight.has(t.id)) return false;
|
|
||||||
if (t.status !== 'ready' && t.status !== 'queued') return false;
|
if (t.status !== 'ready' && t.status !== 'queued') return false;
|
||||||
if (parents.has(t.id)) return false; // 非叶子(容器)跳过
|
if (parents.has(t.id)) return false; // 非叶子(容器)跳过
|
||||||
if (easyOnly && t.complexity !== 'easy') return false;
|
if (easyOnly && t.complexity !== 'easy') return false;
|
||||||
const until = backoffUntil.get(t.id);
|
if (t.nextEligibleAt && nowMs < Date.parse(t.nextEligibleAt)) return false; // 退避冷却中
|
||||||
if (until !== undefined && nowMs < until) return false; // 退避冷却中
|
|
||||||
return t.deps.every((dep) => byId.get(dep)?.status === 'done');
|
return t.deps.every((dep) => byId.get(dep)?.status === 'done');
|
||||||
});
|
});
|
||||||
return rankByScore(candidates, tasks);
|
return rankByScore(candidates, tasks);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 跑一个独立复审 run(kind=reviewer/security)。任何失败折叠为 summary=失败原因、verdict=null,不抛错、不挡任务。 */
|
/** 领取一个任务:建分支/worktree 路径 → executing → startRun → 写 job.json → spawn worker → 记 pid。 */
|
||||||
async function runOneReview(
|
function claimOne(project: Project, task: Task, score: number): void {
|
||||||
kind: 'reviewer' | 'security',
|
|
||||||
fn: ReviewerFn,
|
|
||||||
task: Task,
|
|
||||||
project: Project,
|
|
||||||
wt: WorktreeInfo,
|
|
||||||
executorReport: string,
|
|
||||||
): Promise<{ summary: string | null; verdict: ReviewVerdict | null }> {
|
|
||||||
const label = kind === 'reviewer' ? 'code review' : '安全审计';
|
|
||||||
let reviewRunId: string | null = null;
|
|
||||||
try {
|
|
||||||
const review = store.startRun(task.id, kind, { worktree: wt.dir, branch: wt.branch });
|
|
||||||
reviewRunId = review.id;
|
|
||||||
log.info(`${label} 任务 ${task.id} run=${review.id} model=${pickModel(task, project, 'reviewer')}`);
|
|
||||||
const rv = await fn(task, project, wt, review.id, executorReport);
|
|
||||||
store.finishRun(review.id, 'succeeded', {
|
|
||||||
transcriptRef: rv.transcriptRef ?? undefined,
|
|
||||||
claudeSessionId: rv.sessionId ?? undefined,
|
|
||||||
});
|
|
||||||
log.info(`任务 ${task.id} ${label} 完成 verdict=${rv.verdict ?? '(未解析到)'}`);
|
|
||||||
return { summary: rv.summary, verdict: rv.verdict };
|
|
||||||
} catch (e) {
|
|
||||||
const reMsg = (e as Error).message;
|
|
||||||
if (reviewRunId) {
|
|
||||||
try { store.finishRun(reviewRunId, 'failed', { error: reMsg }); } catch { /* 收尾失败不影响主流程 */ }
|
|
||||||
}
|
|
||||||
log.error(`任务 ${task.id} ${label} 失败(不挡任务,照常进 exec_review):${reMsg}`);
|
|
||||||
return { summary: `自动复审失败:${reMsg}`, verdict: null };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 单任务全流程:executing → worktree → run → verify → code review run → 安全审计 run(均失败不挡)→ setResult(四字段) → exec_review / failed(重试) */
|
|
||||||
async function executeTask(project: Project, task: Task): Promise<void> {
|
|
||||||
let runId: string | null = null;
|
let runId: string | null = null;
|
||||||
let runClosed = false;
|
|
||||||
try {
|
try {
|
||||||
|
if (task.status === 'ready') store.transition(task.id, 'queued', { by: 'orchestrator', score });
|
||||||
|
|
||||||
|
const branch = branchFor(task.id);
|
||||||
|
const dir = worktreeDirFor(project.repoPath, task.id);
|
||||||
|
|
||||||
store.transition(task.id, 'executing', { by: 'orchestrator' });
|
store.transition(task.id, 'executing', { by: 'orchestrator' });
|
||||||
const wt = await d.createWorktree(project.repoPath, task.id, project.defaultBranch);
|
const run = store.startRun(task.id, 'executor', { worktree: dir, branch });
|
||||||
const run = store.startRun(task.id, 'executor', { worktree: wt.dir, branch: wt.branch });
|
|
||||||
runId = run.id;
|
runId = run.id;
|
||||||
log.info(`执行任务 ${task.id}「${task.title}」 run=${run.id} worktree=${wt.dir}`);
|
|
||||||
|
|
||||||
const rr = await d.runner(task, project, wt, run.id);
|
d.writeJobSpec({
|
||||||
if (!rr.ok) {
|
runId: run.id,
|
||||||
store.finishRun(run.id, 'failed', {
|
task: { ...task, status: 'executing' },
|
||||||
error: rr.error ?? '执行失败',
|
project,
|
||||||
transcriptRef: rr.transcriptRef ?? undefined,
|
worktreeDir: dir,
|
||||||
claudeSessionId: rr.sessionId ?? undefined,
|
branch,
|
||||||
});
|
|
||||||
runClosed = true;
|
|
||||||
throw new Error(rr.error ?? '执行失败');
|
|
||||||
}
|
|
||||||
|
|
||||||
const vr = await d.verify(project, wt.dir, run.id);
|
|
||||||
if (!vr.ok) {
|
|
||||||
store.finishRun(run.id, 'failed', {
|
|
||||||
error: vr.error ?? 'verify 失败',
|
|
||||||
transcriptRef: rr.transcriptRef ?? undefined,
|
|
||||||
claudeSessionId: rr.sessionId ?? undefined,
|
|
||||||
});
|
|
||||||
runClosed = true;
|
|
||||||
throw new Error(vr.error ?? 'verify 失败');
|
|
||||||
}
|
|
||||||
|
|
||||||
const diff = await d.worktreeDiff(project.repoPath, wt.dir, wt.branch, project.defaultBranch);
|
|
||||||
|
|
||||||
// 双复审(顺序):code review run(kind=reviewer)→ 安全审计 run(kind=security)。任一失败不挡任务。
|
|
||||||
const code = await runOneReview('reviewer', d.reviewCode, task, project, wt, rr.finalText ?? '');
|
|
||||||
const sec = await runOneReview('security', d.reviewSecurity, task, project, wt, rr.finalText ?? '');
|
|
||||||
|
|
||||||
store.setResult(task.id, {
|
|
||||||
branch: wt.branch, worktree: wt.dir,
|
|
||||||
diffSummary: diff.diffSummary, commits: diff.commits, prUrl: null,
|
|
||||||
summary: code.summary, verdict: code.verdict,
|
|
||||||
securitySummary: sec.summary, securityVerdict: sec.verdict,
|
|
||||||
mergeTaskId: null,
|
|
||||||
});
|
});
|
||||||
store.transition(task.id, 'exec_review', { by: 'orchestrator', runId: run.id });
|
|
||||||
store.finishRun(run.id, 'succeeded', {
|
log.info(`领取任务 ${task.id}「${task.title}」score=${score} run=${run.id} model=${pickModel(task, project, 'executor')}`);
|
||||||
transcriptRef: rr.transcriptRef ?? undefined,
|
const pid = d.spawnWorker(run.id);
|
||||||
claudeSessionId: rr.sessionId ?? undefined,
|
store.setWorkerPid(run.id, pid);
|
||||||
});
|
log.info(`任务 ${task.id} 已起 worker pid=${pid} worktree=${dir}`);
|
||||||
log.info(`任务 ${task.id} 执行完成 → exec_review(${diff.commits.length} commits)`);
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const msg = (e as Error).message;
|
const msg = (e as Error).message;
|
||||||
log.error(`任务 ${task.id} 执行失败:${msg}`);
|
log.error(`任务 ${task.id} 领取/spawn 失败:${msg}`);
|
||||||
try {
|
try {
|
||||||
if (runId && !runClosed) {
|
store.failTaskAttempt(task.id, runId, msg);
|
||||||
store.finishRun(runId, 'failed', { error: msg }); // worktree 创建后抛错(diff 等)时收尾
|
|
||||||
} else if (!runId) {
|
|
||||||
// run 还没建(如 createWorktree 失败):补记一条 failed run,保证重试计数不漏
|
|
||||||
const r = store.startRun(task.id, 'executor');
|
|
||||||
store.finishRun(r.id, 'failed', { error: msg });
|
|
||||||
}
|
|
||||||
store.transition(task.id, 'failed', { by: 'orchestrator', error: msg });
|
|
||||||
const allFailed = store.listRuns(task.id).filter((r) => r.kind === 'executor' && r.status === 'failed').length;
|
|
||||||
// 相对于上次手动重投基线的净失败次数(retryBaseline 在 requeueTask 时设置)
|
|
||||||
const netFailed = Math.max(0, allFailed - task.retryBaseline);
|
|
||||||
const priorNetFailed = Math.max(0, netFailed - 1); // 不含本次
|
|
||||||
const maxRetries = project.maxRetries;
|
|
||||||
if (priorNetFailed < maxRetries) {
|
|
||||||
const attempt = priorNetFailed + 1; // 本次是第几次重试
|
|
||||||
const backoffMs = computeBackoffMs(attempt);
|
|
||||||
backoffUntil.set(task.id, d.nowMs() + backoffMs);
|
|
||||||
store.transition(task.id, 'queued', { by: 'orchestrator', retry: attempt, backoffMs });
|
|
||||||
log.info(`任务 ${task.id} 重新入队(第 ${attempt} 次重试,退避 ${backoffMs}ms 后可领)`);
|
|
||||||
} else {
|
|
||||||
backoffUntil.delete(task.id);
|
|
||||||
store.transition(task.id, 'needs_attention', { by: 'orchestrator', allFailed, netFailed });
|
|
||||||
log.error(`任务 ${task.id} 净失败 ${netFailed} 次(上限 ${maxRetries})→ needs_attention`);
|
|
||||||
}
|
|
||||||
} catch (e2) {
|
} catch (e2) {
|
||||||
log.error(`任务 ${task.id} 失败收尾出错:${(e2 as Error).message}`);
|
log.error(`任务 ${task.id} 失败收尾出错:${(e2 as Error).message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function tick(): void {
|
/** 领取轮:对每个 active 且 autonomy≠manual 的项目,在 countExecuting<concurrency 时按 claimable 领新任务并 spawn worker。 */
|
||||||
|
function claimTick(): void {
|
||||||
try {
|
try {
|
||||||
for (const p of store.listProjects()) {
|
for (const p of store.listProjects()) {
|
||||||
if (p.status !== 'active' || p.autonomy === 'manual') continue;
|
if (p.status !== 'active' || p.autonomy === 'manual') continue;
|
||||||
let active = 0;
|
let active = store.countExecuting(p.id);
|
||||||
for (const pid of inflight.values()) if (pid === p.id) active++;
|
|
||||||
if (active >= p.concurrency) continue;
|
if (active >= p.concurrency) continue;
|
||||||
|
|
||||||
for (const { task, score } of claimable(p)) {
|
for (const { task, score } of claimable(p)) {
|
||||||
if (active >= p.concurrency) break;
|
if (active >= p.concurrency) break;
|
||||||
try {
|
claimOne(p, task, score);
|
||||||
if (task.status === 'ready') store.transition(task.id, 'queued', { by: 'orchestrator', score });
|
|
||||||
} catch (e) {
|
|
||||||
log.error(`任务 ${task.id} 入队失败:${(e as Error).message}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
log.info(`领取任务 ${task.id}「${task.title}」score=${score} model=${pickModel(task, p, 'executor')}`);
|
|
||||||
inflight.set(task.id, p.id);
|
|
||||||
active++;
|
active++;
|
||||||
const job: Promise<void> = executeTask(p, { ...task, status: 'queued' })
|
|
||||||
.catch((e) => log.error(`任务 ${task.id} 执行异常:${(e as Error).message}`))
|
|
||||||
.finally(() => {
|
|
||||||
inflight.delete(task.id);
|
|
||||||
pending.delete(job);
|
|
||||||
});
|
|
||||||
pending.add(job);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
log.error(`编排器轮询失败:${(e as Error).message}`);
|
log.error(`编排器领取轮失败:${(e as Error).message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function drain(): Promise<void> {
|
/** ingest 所有在途 run 的 outbox(daemon 唯一 DB 写者把 worker 进度/结果落库)。 */
|
||||||
while (pending.size > 0) await Promise.allSettled([...pending]);
|
function ingest(): void {
|
||||||
|
try {
|
||||||
|
d.ingestAll(store, log);
|
||||||
|
} catch (e) {
|
||||||
|
log.error(`ingest 轮失败:${(e as Error).message}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { tick, drain, inflight };
|
/** 回收:对每个 executing 任务判活,死 worker → failTaskAttempt(收尾 + 重试/needs_attention)。 */
|
||||||
|
function reap(): void {
|
||||||
|
try {
|
||||||
|
for (const { task, run } of store.executingWithLatestExecutorRun()) {
|
||||||
|
const alive = run !== null && d.isWorkerAlive({
|
||||||
|
pid: run.workerPid,
|
||||||
|
heartbeatAgeMs: heartbeatAgeMs(run.id, d.nowMs()),
|
||||||
|
startedAgeMs: d.nowMs() - Date.parse(run.startedAt),
|
||||||
|
});
|
||||||
|
if (alive) continue;
|
||||||
|
log.error(`任务 ${task.id} worker 异常退出(run=${run?.id ?? '无'})→ 回收`);
|
||||||
|
try {
|
||||||
|
store.failTaskAttempt(task.id, run?.id ?? null, 'worker 异常退出');
|
||||||
|
} catch (e) {
|
||||||
|
log.error(`任务 ${task.id} 回收收尾出错:${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
log.error(`编排器回收轮失败:${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function tick(): void {
|
||||||
|
ingest(); // 先把 worker 进度/结果落库(可能把 executing→exec_review,腾出并发槽)
|
||||||
|
reap(); // 再回收死 worker(可能把 executing→queued/needs_attention,腾出并发槽)
|
||||||
|
claimTick(); // 最后领新任务
|
||||||
|
}
|
||||||
|
|
||||||
|
return { tick, claimTick, ingest, reap };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 接线入口:MAESTRO_ORCH_INTERVAL(秒)控制轮询间隔,默认 15,0=关闭。
|
* 接线入口:MAESTRO_ORCH_INTERVAL(秒)控制轮询间隔,默认 15,0=关闭。
|
||||||
* 返回 timer 供 shutdown 时 clearInterval。
|
* 每轮依次 ingest → reaper → 领取(见 tick)。返回 timer 供 shutdown 时 clearInterval。
|
||||||
*/
|
*/
|
||||||
export function startOrchestrator(
|
export function startOrchestrator(
|
||||||
store: Store,
|
store: Store,
|
||||||
@@ -255,6 +231,6 @@ export function startOrchestrator(
|
|||||||
const orch = createOrchestrator(store, app.log, deps);
|
const orch = createOrchestrator(store, app.log, deps);
|
||||||
const timer = setInterval(() => orch.tick(), intervalSec * 1000);
|
const timer = setInterval(() => orch.tick(), intervalSec * 1000);
|
||||||
timer.unref();
|
timer.unref();
|
||||||
app.log.info(`编排器已启用:每 ${intervalSec}s 一轮领取(autonomy≠manual 的 active 项目)`);
|
app.log.info(`编排器已启用:每 ${intervalSec}s 一轮(ingest→回收→领取,autonomy≠manual 的 active 项目)`);
|
||||||
return timer;
|
return timer;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { mkdtempSync, rmSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { Store } from '../src/store/index.js';
|
||||||
|
import { ingestRun } from '../src/daemon/ingest.js';
|
||||||
|
import { appendOutbox, type OutboxPayload } from '../src/executor/protocol.js';
|
||||||
|
import { branchFor, worktreeDirFor } from '../src/executor/worktree.js';
|
||||||
|
|
||||||
|
const noopLog = { info: (): void => undefined, error: (): void => undefined };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 隔离的 MAESTRO_DATA_DIR(outbox 落在临时目录),跑回调,结束清理。
|
||||||
|
* protocol.ts 的 runDir/outboxPath 在调用时读 env,故进 cb 前设好、cb 内的 append/ingest 都命中临时目录。
|
||||||
|
*/
|
||||||
|
function withTmpDataDir(cb: () => void): void {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'maestro-ingest-'));
|
||||||
|
const prev = process.env.MAESTRO_DATA_DIR;
|
||||||
|
process.env.MAESTRO_DATA_DIR = dir;
|
||||||
|
try {
|
||||||
|
cb();
|
||||||
|
} finally {
|
||||||
|
if (prev === undefined) delete process.env.MAESTRO_DATA_DIR;
|
||||||
|
else process.env.MAESTRO_DATA_DIR = prev;
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 真 Store + 项目 + easy 任务推进到 executing,并建一条 executor run(模拟 daemon 已领取、worker 已 spawn)。 */
|
||||||
|
function setupExecuting(opts: { autonomy?: 'auto-easy' | 'auto-approved'; maxRetries?: number } = {}): {
|
||||||
|
store: Store; projectId: string; taskId: string; runId: string; dir: string; branch: string;
|
||||||
|
} {
|
||||||
|
const store = new Store(':memory:');
|
||||||
|
const p = store.createProject({
|
||||||
|
name: 'ingest', repoPath: '/tmp/ingest-repo-' + Math.random(),
|
||||||
|
autonomy: opts.autonomy ?? 'auto-easy', maxRetries: opts.maxRetries,
|
||||||
|
});
|
||||||
|
const t = store.createTask({ projectId: p.id, title: 'tweak', complexity: 'easy' });
|
||||||
|
store.setOperations(t.id, '在 README.md 追加一行');
|
||||||
|
const branch = branchFor(t.id);
|
||||||
|
const dir = worktreeDirFor(p.repoPath, t.id);
|
||||||
|
store.transition(t.id, 'queued', { by: 'test' });
|
||||||
|
store.transition(t.id, 'executing', { by: 'test' });
|
||||||
|
const run = store.startRun(t.id, 'executor', { worktree: dir, branch });
|
||||||
|
store.setWorkerPid(run.id, 4242);
|
||||||
|
return { store, projectId: p.id, taskId: t.id, runId: run.id, dir, branch };
|
||||||
|
}
|
||||||
|
|
||||||
|
function resultPayload(branch: string, worktree: string): OutboxPayload {
|
||||||
|
return {
|
||||||
|
type: 'result',
|
||||||
|
branch, worktree,
|
||||||
|
diffSummary: ' README.md | 1 +',
|
||||||
|
commits: ['abc1234 hello maestro'],
|
||||||
|
executor: { transcriptRef: '/tmp/exec.jsonl', sessionId: 'sess-exec-1' },
|
||||||
|
code: { summary: '## 做了什么\nmock 复审通过', verdict: 'approve', transcriptRef: '/tmp/code.jsonl' },
|
||||||
|
security: { summary: '## 安全审计\nmock 审计通过', verdict: 'approve', transcriptRef: '/tmp/sec.jsonl' },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('ingest result:task→exec_review,setResult 四字段正确,reviewer/security/executor 各 1 条 succeeded', () => {
|
||||||
|
withTmpDataDir(() => {
|
||||||
|
const { store, taskId, runId, dir, branch } = setupExecuting();
|
||||||
|
|
||||||
|
appendOutbox(runId, { type: 'started', pid: 4242, worktree: dir, branch, model: 'claude-sonnet-4-6' });
|
||||||
|
appendOutbox(runId, { type: 'phase', phase: 'executing' });
|
||||||
|
appendOutbox(runId, resultPayload(branch, dir));
|
||||||
|
appendOutbox(runId, { type: 'done' });
|
||||||
|
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
|
||||||
|
const done = store.getTask(taskId)!;
|
||||||
|
assert.equal(done.status, 'exec_review');
|
||||||
|
assert.deepEqual(done.result, {
|
||||||
|
branch,
|
||||||
|
worktree: dir,
|
||||||
|
diffSummary: ' README.md | 1 +',
|
||||||
|
commits: ['abc1234 hello maestro'],
|
||||||
|
prUrl: null,
|
||||||
|
summary: '## 做了什么\nmock 复审通过',
|
||||||
|
verdict: 'approve',
|
||||||
|
securitySummary: '## 安全审计\nmock 审计通过',
|
||||||
|
securityVerdict: 'approve',
|
||||||
|
mergeTaskId: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const runs = store.listRuns(taskId);
|
||||||
|
const executor = runs.find((r) => r.kind === 'executor')!;
|
||||||
|
assert.equal(executor.status, 'succeeded');
|
||||||
|
assert.equal(executor.transcriptRef, '/tmp/exec.jsonl');
|
||||||
|
assert.equal(executor.claudeSessionId, 'sess-exec-1');
|
||||||
|
const reviewer = runs.filter((r) => r.kind === 'reviewer');
|
||||||
|
assert.equal(reviewer.length, 1);
|
||||||
|
assert.equal(reviewer[0].status, 'succeeded');
|
||||||
|
assert.equal(reviewer[0].transcriptRef, '/tmp/code.jsonl');
|
||||||
|
const security = runs.filter((r) => r.kind === 'security');
|
||||||
|
assert.equal(security.length, 1);
|
||||||
|
assert.equal(security[0].status, 'succeeded');
|
||||||
|
assert.equal(security[0].transcriptRef, '/tmp/sec.jsonl');
|
||||||
|
|
||||||
|
// 游标推进到末条(done.seq=4)
|
||||||
|
assert.equal(store.getRun(runId)!.lastSeq, 4);
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ingest result:任一 verdict=reject 也照常落进 result(裁决归用户)', () => {
|
||||||
|
withTmpDataDir(() => {
|
||||||
|
const { store, taskId, runId, dir, branch } = setupExecuting();
|
||||||
|
appendOutbox(runId, {
|
||||||
|
...resultPayload(branch, dir),
|
||||||
|
code: { summary: '发现问题', verdict: 'reject', transcriptRef: '/tmp/code.jsonl' },
|
||||||
|
security: { summary: '发现密钥泄露', verdict: 'reject', transcriptRef: '/tmp/sec.jsonl' },
|
||||||
|
} as OutboxPayload);
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
|
||||||
|
const done = store.getTask(taskId)!;
|
||||||
|
assert.equal(done.status, 'exec_review');
|
||||||
|
assert.equal(done.result!.verdict, 'reject');
|
||||||
|
assert.equal(done.result!.summary, '发现问题');
|
||||||
|
assert.equal(done.result!.securityVerdict, 'reject');
|
||||||
|
assert.equal(done.result!.securitySummary, '发现密钥泄露');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ingest failed:首次失败 → task 转 queued + 持久化退避(nextEligibleAt 有值)+ executor run failed', () => {
|
||||||
|
withTmpDataDir(() => {
|
||||||
|
const { store, taskId, runId } = setupExecuting();
|
||||||
|
|
||||||
|
appendOutbox(runId, { type: 'started', pid: 4242, worktree: '/x', branch: 'b', model: 'm' });
|
||||||
|
appendOutbox(runId, { type: 'failed', error: 'boom #1', transcriptRef: '/tmp/f.jsonl', sessionId: 'sess-f-1' });
|
||||||
|
appendOutbox(runId, { type: 'done' });
|
||||||
|
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
|
||||||
|
const t = store.getTask(taskId)!;
|
||||||
|
assert.equal(t.status, 'queued'); // 第一次失败 → 重入队(默认 maxRetries=2)
|
||||||
|
assert.ok(t.nextEligibleAt, 'nextEligibleAt 应被持久化(退避)');
|
||||||
|
assert.ok(Date.parse(t.nextEligibleAt!) > Date.now() - 1000, 'nextEligibleAt 在未来');
|
||||||
|
|
||||||
|
const executor = store.listRuns(taskId).find((r) => r.kind === 'executor')!;
|
||||||
|
assert.equal(executor.status, 'failed');
|
||||||
|
assert.match(executor.error ?? '', /boom #1/);
|
||||||
|
assert.equal(executor.transcriptRef, '/tmp/f.jsonl');
|
||||||
|
assert.equal(executor.claudeSessionId, 'sess-f-1');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ingest failed:maxRetries=0 → 首次失败直接 needs_attention', () => {
|
||||||
|
withTmpDataDir(() => {
|
||||||
|
const { store, taskId, runId } = setupExecuting({ maxRetries: 0 });
|
||||||
|
appendOutbox(runId, { type: 'failed', error: 'boom', transcriptRef: null, sessionId: null });
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
assert.equal(store.getTask(taskId)!.status, 'needs_attention');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('幂等:同一批 outbox ingest 两次,DB 不重复变更(lastSeq 生效)', () => {
|
||||||
|
withTmpDataDir(() => {
|
||||||
|
const { store, taskId, runId, dir, branch } = setupExecuting();
|
||||||
|
appendOutbox(runId, { type: 'started', pid: 4242, worktree: dir, branch, model: 'm' });
|
||||||
|
appendOutbox(runId, resultPayload(branch, dir));
|
||||||
|
appendOutbox(runId, { type: 'done' });
|
||||||
|
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
const after1 = store.getTask(taskId)!;
|
||||||
|
const runs1 = store.listRuns(taskId);
|
||||||
|
assert.equal(after1.status, 'exec_review');
|
||||||
|
assert.equal(runs1.filter((r) => r.kind === 'reviewer').length, 1);
|
||||||
|
assert.equal(runs1.filter((r) => r.kind === 'security').length, 1);
|
||||||
|
const lastSeq1 = store.getRun(runId)!.lastSeq;
|
||||||
|
|
||||||
|
// 第二次 ingest:seq 全部 ≤ lastSeq → 不处理任何记录
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
const runs2 = store.listRuns(taskId);
|
||||||
|
assert.equal(runs2.length, runs1.length, '复审 run 不应重复创建');
|
||||||
|
assert.equal(runs2.filter((r) => r.kind === 'reviewer').length, 1);
|
||||||
|
assert.equal(runs2.filter((r) => r.kind === 'security').length, 1);
|
||||||
|
assert.equal(store.getRun(runId)!.lastSeq, lastSeq1, 'lastSeq 不变');
|
||||||
|
assert.equal(store.getTask(taskId)!.status, 'exec_review');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ingest 续读:先 ingest 半截(started/phase),再追加 result,第二次 ingest 完成 exec_review', () => {
|
||||||
|
withTmpDataDir(() => {
|
||||||
|
const { store, taskId, runId, dir, branch } = setupExecuting();
|
||||||
|
appendOutbox(runId, { type: 'started', pid: 4242, worktree: dir, branch, model: 'm' });
|
||||||
|
appendOutbox(runId, { type: 'phase', phase: 'executing' });
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
assert.equal(store.getTask(taskId)!.status, 'executing'); // 还没到 result,仍 executing
|
||||||
|
assert.equal(store.getRun(runId)!.lastSeq, 2);
|
||||||
|
|
||||||
|
appendOutbox(runId, resultPayload(branch, dir));
|
||||||
|
appendOutbox(runId, { type: 'done' });
|
||||||
|
ingestRun(store, noopLog, runId);
|
||||||
|
assert.equal(store.getTask(taskId)!.status, 'exec_review');
|
||||||
|
assert.equal(store.getRun(runId)!.lastSeq, 4);
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
+292
-398
@@ -1,53 +1,44 @@
|
|||||||
import { test } from 'node:test';
|
import { test } from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { Store } from '../src/store/index.js';
|
import { Store } from '../src/store/index.js';
|
||||||
import { createOrchestrator, DEFAULT_MAX_RETRIES, computeBackoffMs, type OrchestratorDeps } from '../src/daemon/orchestrator.js';
|
import {
|
||||||
import type { RunnerResult } from '../src/executor/runner.js';
|
createOrchestrator, workerEntry, workerEnv, DEFAULT_MAX_RETRIES,
|
||||||
import type { ReviewResult } from '../src/executor/reviewer.js';
|
type OrchestratorDeps,
|
||||||
|
} from '../src/daemon/orchestrator.js';
|
||||||
|
import type { JobSpec } from '../src/executor/protocol.js';
|
||||||
import type { Autonomy } from '../src/model/types.js';
|
import type { Autonomy } from '../src/model/types.js';
|
||||||
|
|
||||||
const noopLog = { info: (): void => undefined, error: (): void => undefined };
|
const noopLog = { info: (): void => undefined, error: (): void => undefined };
|
||||||
|
|
||||||
function deferred<T>(): { promise: Promise<T>; resolve: (v: T) => void } {
|
/** 可前进的测试时钟。 */
|
||||||
let resolve!: (v: T) => void;
|
function makeClock(initialMs = Date.now()): { nowMs: () => number; advance: (ms: number) => void } {
|
||||||
const promise = new Promise<T>((r) => { resolve = r; });
|
let t = initialMs;
|
||||||
return { promise, resolve };
|
return { nowMs: () => t, advance: (ms) => { t += ms; } };
|
||||||
}
|
}
|
||||||
|
|
||||||
const okRun: RunnerResult = { ok: true, transcriptRef: '/tmp/fake.jsonl', sessionId: 'sess-mock-1', finalText: '执行自述:改了 README' };
|
interface MockState {
|
||||||
|
spawned: string[]; // 被 spawn 的 runId 列表
|
||||||
|
jobs: JobSpec[]; // 被 writeJobSpec 的 job 列表
|
||||||
|
ingestCalls: number;
|
||||||
|
alive: boolean; // isWorkerAlive 的返回(reaper 用)
|
||||||
|
nextPid: number;
|
||||||
|
}
|
||||||
|
|
||||||
const okReview: ReviewResult = {
|
/** 全 mock 依赖:不真 spawn / 不真判活 / ingest no-op / job 不落盘。可按用例覆盖。 */
|
||||||
summary: '## 做了什么\nmock 复审通过',
|
function mockDeps(state: MockState, overrides: Partial<OrchestratorDeps> = {}): { deps: OrchestratorDeps } {
|
||||||
verdict: 'approve',
|
const deps: OrchestratorDeps = {
|
||||||
transcriptRef: '/tmp/fake-review.jsonl',
|
spawnWorker: (runId: string): number => { state.spawned.push(runId); return state.nextPid++; },
|
||||||
sessionId: 'sess-review-1',
|
isWorkerAlive: () => state.alive,
|
||||||
};
|
ingestAll: () => { state.ingestCalls++; },
|
||||||
|
writeJobSpec: (job: JobSpec) => { state.jobs.push(job); },
|
||||||
const okSecurity: ReviewResult = {
|
|
||||||
summary: '## 安全审计\nmock 审计通过',
|
|
||||||
verdict: 'approve',
|
|
||||||
transcriptRef: '/tmp/fake-security.jsonl',
|
|
||||||
sessionId: 'sess-security-1',
|
|
||||||
};
|
|
||||||
|
|
||||||
/** 全 mock 依赖(不真起 CC、不动 git):可按用例覆盖 */
|
|
||||||
function mockDeps(overrides: Partial<OrchestratorDeps> = {}): OrchestratorDeps {
|
|
||||||
return {
|
|
||||||
createWorktree: async (_repo, taskId) => ({ dir: `/tmp/fake-wt/${taskId}`, branch: `maestro/${taskId}` }),
|
|
||||||
worktreeDiff: async () => ({ diffSummary: ' README.md | 1 +', commits: ['abc1234 hello maestro'] }),
|
|
||||||
verify: async () => ({ ok: true, exitCode: 0, logRef: null }),
|
|
||||||
runner: async () => okRun,
|
|
||||||
reviewCode: async () => okReview,
|
|
||||||
reviewSecurity: async () => okSecurity,
|
|
||||||
nowMs: Date.now,
|
nowMs: Date.now,
|
||||||
...overrides,
|
...overrides,
|
||||||
};
|
};
|
||||||
|
return { deps };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 可前进的测试时钟(用于退避相关测试) */
|
function freshState(): MockState {
|
||||||
function makeClock(initialMs = 0): { nowMs: () => number; advance: (ms: number) => void } {
|
return { spawned: [], jobs: [], ingestCalls: 0, alive: true, nextPid: 1000 };
|
||||||
let t = initialMs;
|
|
||||||
return { nowMs: () => t, advance: (ms) => { t += ms; } };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
|
function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId: string } {
|
||||||
@@ -58,25 +49,139 @@ function setup(autonomy: Autonomy, concurrency = 1): { store: Store; projectId:
|
|||||||
return { store, projectId: p.id };
|
return { store, projectId: p.id };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function settle(): Promise<void> {
|
// ───────────────────────── workerEntry / workerEnv(可读测的小函数)─────────────────────────
|
||||||
await new Promise((r) => setImmediate(r));
|
|
||||||
}
|
|
||||||
|
|
||||||
test('autonomy=manual:编排器不领取任何任务', async () => {
|
test('workerEntry:默认指向 ../executor/worker.js(node 运行)', () => {
|
||||||
const { store, projectId } = setup('manual');
|
const prev = process.env.MAESTRO_WORKER_CMD;
|
||||||
const t = store.createTask({ projectId, title: 'easy task', complexity: 'easy' });
|
delete process.env.MAESTRO_WORKER_CMD;
|
||||||
let calls = 0;
|
const [cmd, scriptPath] = workerEntry();
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
assert.equal(cmd, 'node');
|
||||||
runner: async () => { calls++; return okRun; },
|
assert.match(scriptPath, /executor[/\\]worker\.js$/);
|
||||||
}));
|
if (prev !== undefined) process.env.MAESTRO_WORKER_CMD = prev;
|
||||||
|
});
|
||||||
|
|
||||||
|
test('workerEntry:MAESTRO_WORKER_CMD 覆盖整条命令(空格分隔)', () => {
|
||||||
|
const prev = process.env.MAESTRO_WORKER_CMD;
|
||||||
|
process.env.MAESTRO_WORKER_CMD = 'npx tsx src/executor/worker.ts';
|
||||||
|
assert.deepEqual(workerEntry(), ['npx', 'tsx', 'src/executor/worker.ts']);
|
||||||
|
if (prev === undefined) delete process.env.MAESTRO_WORKER_CMD;
|
||||||
|
else process.env.MAESTRO_WORKER_CMD = prev;
|
||||||
|
});
|
||||||
|
|
||||||
|
test('workerEnv:只透传 PATH/HOME/LANG + ANTHROPIC_/CLAUDE_ 前缀;丢弃无关与 MAESTRO_ 控制变量', () => {
|
||||||
|
const env = workerEnv({
|
||||||
|
PATH: '/usr/bin', HOME: '/home/u', LANG: 'en_US.UTF-8',
|
||||||
|
ANTHROPIC_API_KEY: 'sk-x', CLAUDE_CODE_FOO: 'y',
|
||||||
|
MAESTRO_ORCH_INTERVAL: '15', MAESTRO_DATA_DIR: '/data',
|
||||||
|
SOME_SECRET: 'leak', AWS_SECRET_ACCESS_KEY: 'nope',
|
||||||
|
});
|
||||||
|
assert.equal(env.PATH, '/usr/bin');
|
||||||
|
assert.equal(env.HOME, '/home/u');
|
||||||
|
assert.equal(env.LANG, 'en_US.UTF-8');
|
||||||
|
assert.equal(env.ANTHROPIC_API_KEY, 'sk-x');
|
||||||
|
assert.equal(env.CLAUDE_CODE_FOO, 'y');
|
||||||
|
assert.equal(env.MAESTRO_DATA_DIR, '/data'); // 例外:worker 须看同一 runs/ 目录
|
||||||
|
assert.equal(env.MAESTRO_ORCH_INTERVAL, undefined, 'daemon 控制变量不下传');
|
||||||
|
assert.equal(env.SOME_SECRET, undefined, '无关变量不下传');
|
||||||
|
assert.equal(env.AWS_SECRET_ACCESS_KEY, undefined, '无关密钥不下传');
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────── tick 顺序:ingest + reaper 先于领取 ─────────────────────────
|
||||||
|
|
||||||
|
test('tick:每轮先 ingest 再 reaper 再领取', () => {
|
||||||
|
const { store, projectId } = setup('auto-easy');
|
||||||
|
store.createTask({ projectId, title: 'a', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
orch.tick();
|
orch.tick();
|
||||||
await orch.drain();
|
assert.equal(state.ingestCalls, 1, '每轮 ingest 一次');
|
||||||
assert.equal(calls, 0);
|
assert.equal(state.spawned.length, 1, '领取并 spawn 一个');
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────── 领取(监工)─────────────────────────
|
||||||
|
|
||||||
|
test('autonomy=manual:不领取', () => {
|
||||||
|
const { store, projectId } = setup('manual');
|
||||||
|
const t = store.createTask({ projectId, title: 'easy', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
createOrchestrator(store, noopLog, deps).tick();
|
||||||
|
assert.equal(state.spawned.length, 0);
|
||||||
assert.equal(store.getTask(t.id)!.status, 'ready');
|
assert.equal(store.getTask(t.id)!.status, 'ready');
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('autonomy=auto-easy:只领 easy,medium ready 不动', async () => {
|
test('project paused:不领取', () => {
|
||||||
|
const { store, projectId } = setup('auto-approved');
|
||||||
|
const t = store.createTask({ projectId, title: 'x', complexity: 'easy' });
|
||||||
|
store.patchProject(projectId, { status: 'paused' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
createOrchestrator(store, noopLog, deps).tick();
|
||||||
|
assert.equal(state.spawned.length, 0);
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'ready');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('并发闸 concurrency=1:一轮只领一个;领取后 executing + 建 executor run + setWorkerPid + writeJobSpec', () => {
|
||||||
|
const { store, projectId } = setup('auto-approved', 1);
|
||||||
|
const t1 = store.createTask({ projectId, title: 'a', complexity: 'easy' });
|
||||||
|
const t2 = store.createTask({ projectId, title: 'b', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
|
|
||||||
|
orch.claimTick();
|
||||||
|
// 并发=1:只领一个
|
||||||
|
assert.equal(state.spawned.length, 1);
|
||||||
|
assert.equal(state.jobs.length, 1);
|
||||||
|
|
||||||
|
// 被领的那个进 executing,另一个仍 ready
|
||||||
|
const a = store.getTask(t1.id)!;
|
||||||
|
const b = store.getTask(t2.id)!;
|
||||||
|
const executing = a.status === 'executing' ? a : b;
|
||||||
|
const stillReady = a.status === 'executing' ? b : a;
|
||||||
|
assert.equal(executing.status, 'executing');
|
||||||
|
assert.equal(stillReady.status, 'ready');
|
||||||
|
|
||||||
|
// 建了 executor run + setWorkerPid + writeJobSpec 内容正确
|
||||||
|
const runs = store.listRuns(executing.id);
|
||||||
|
assert.equal(runs.length, 1);
|
||||||
|
const run = runs[0];
|
||||||
|
assert.equal(run.kind, 'executor');
|
||||||
|
assert.equal(run.status, 'started');
|
||||||
|
assert.equal(run.branch, `maestro/${executing.id}`);
|
||||||
|
assert.ok(run.workerPid, 'setWorkerPid 已写 pid');
|
||||||
|
assert.equal(state.spawned[0], run.id, 'spawn 的 runId == 新建 executor run');
|
||||||
|
|
||||||
|
const job = state.jobs[0];
|
||||||
|
assert.equal(job.runId, run.id);
|
||||||
|
assert.equal(job.task.id, executing.id);
|
||||||
|
assert.equal(job.task.status, 'executing');
|
||||||
|
assert.equal(job.branch, `maestro/${executing.id}`);
|
||||||
|
assert.equal(job.worktreeDir, run.worktree);
|
||||||
|
|
||||||
|
// 槽位占满 → 下一轮不再领(仍只 spawn 过一个)
|
||||||
|
orch.claimTick();
|
||||||
|
assert.equal(state.spawned.length, 1, '并发=1,executing 占满 → 不再领');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('并发 concurrency=2:一轮领满两个', () => {
|
||||||
|
const { store, projectId } = setup('auto-approved', 2);
|
||||||
|
store.createTask({ projectId, title: 'a', complexity: 'easy' });
|
||||||
|
store.createTask({ projectId, title: 'b', complexity: 'easy' });
|
||||||
|
store.createTask({ projectId, title: 'c', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
createOrchestrator(store, noopLog, deps).claimTick();
|
||||||
|
assert.equal(state.spawned.length, 2, '并发=2 一轮领两个');
|
||||||
|
assert.equal(store.listTasks(projectId).filter((t) => t.status === 'executing').length, 2);
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('auto-easy:只领 easy,medium ready 不动', () => {
|
||||||
const { store, projectId } = setup('auto-easy', 5);
|
const { store, projectId } = setup('auto-easy', 5);
|
||||||
const easy = store.createTask({ projectId, title: 'small', complexity: 'easy' });
|
const easy = store.createTask({ projectId, title: 'small', complexity: 'easy' });
|
||||||
const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
|
const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
|
||||||
@@ -85,392 +190,181 @@ test('autonomy=auto-easy:只领 easy,medium ready 不动', async () => {
|
|||||||
store.decide(medium.id, 'accept', 'user'); // medium → ready
|
store.decide(medium.id, 'accept', 'user'); // medium → ready
|
||||||
assert.equal(store.getTask(medium.id)!.status, 'ready');
|
assert.equal(store.getTask(medium.id)!.status, 'ready');
|
||||||
|
|
||||||
const ran: string[] = [];
|
const state = freshState();
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
const { deps } = mockDeps(state);
|
||||||
runner: async (task) => { ran.push(task.id); return okRun; },
|
createOrchestrator(store, noopLog, deps).claimTick();
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
|
|
||||||
assert.deepEqual(ran, [easy.id]);
|
assert.equal(state.spawned.length, 1);
|
||||||
assert.equal(store.getTask(easy.id)!.status, 'exec_review');
|
assert.equal(store.getTask(easy.id)!.status, 'executing');
|
||||||
assert.equal(store.getTask(medium.id)!.status, 'ready'); // 不自动跑
|
assert.equal(store.getTask(medium.id)!.status, 'ready'); // 不领 medium
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('autonomy=auto-approved:领全部 ready(含 medium),依赖未满足/非叶子不领', async () => {
|
test('auto-approved:领 ready(含 medium);依赖未满足/非叶子不领', () => {
|
||||||
const { store, projectId } = setup('auto-approved', 5);
|
const { store, projectId } = setup('auto-approved', 5);
|
||||||
const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
|
const medium = store.createTask({ projectId, title: 'mid', complexity: 'medium' });
|
||||||
store.setSpec(medium.id, '方案');
|
store.setSpec(medium.id, '方案');
|
||||||
store.transition(medium.id, 'spec_review');
|
store.transition(medium.id, 'spec_review');
|
||||||
store.decide(medium.id, 'accept', 'user');
|
store.decide(medium.id, 'accept', 'user');
|
||||||
// 依赖未 done 的任务:建在 medium 上 → blocked,不可领
|
|
||||||
const dep = store.createTask({ projectId, title: 'after-mid', complexity: 'easy', deps: [medium.id] });
|
const dep = store.createTask({ projectId, title: 'after-mid', complexity: 'easy', deps: [medium.id] });
|
||||||
assert.equal(store.getTask(dep.id)!.status, 'blocked');
|
assert.equal(store.getTask(dep.id)!.status, 'blocked');
|
||||||
|
|
||||||
const ran: string[] = [];
|
const state = freshState();
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
const { deps } = mockDeps(state);
|
||||||
runner: async (task) => { ran.push(task.id); return okRun; },
|
createOrchestrator(store, noopLog, deps).claimTick();
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
|
|
||||||
assert.deepEqual(ran, [medium.id]);
|
assert.equal(state.spawned.length, 1);
|
||||||
assert.equal(store.getTask(medium.id)!.status, 'exec_review');
|
assert.equal(store.getTask(medium.id)!.status, 'executing');
|
||||||
assert.equal(store.getTask(dep.id)!.status, 'blocked');
|
assert.equal(store.getTask(dep.id)!.status, 'blocked');
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('concurrency=1:同项目同轮只领 1 个,跑完下一轮再领', async () => {
|
test('退避:nextEligibleAt 在未来 → 不领;过期后 → 领', () => {
|
||||||
const { store, projectId } = setup('auto-approved', 1);
|
|
||||||
const t1 = store.createTask({ projectId, title: 'a', complexity: 'easy' });
|
|
||||||
const t2 = store.createTask({ projectId, title: 'b', complexity: 'easy' });
|
|
||||||
|
|
||||||
const gate = deferred<void>();
|
|
||||||
const started: string[] = [];
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
runner: async (task) => { started.push(task.id); await gate.promise; return okRun; },
|
|
||||||
}));
|
|
||||||
|
|
||||||
orch.tick();
|
|
||||||
await settle();
|
|
||||||
assert.deepEqual(started, [t1.id], '并发=1 只应启动第一个任务');
|
|
||||||
assert.equal(store.getTask(t1.id)!.status, 'executing');
|
|
||||||
assert.equal(store.getTask(t2.id)!.status, 'ready');
|
|
||||||
assert.equal(orch.inflight.size, 1);
|
|
||||||
|
|
||||||
orch.tick(); // 在途占满 → 本轮不领
|
|
||||||
await settle();
|
|
||||||
assert.deepEqual(started, [t1.id]);
|
|
||||||
|
|
||||||
gate.resolve();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t1.id)!.status, 'exec_review');
|
|
||||||
|
|
||||||
orch.tick(); // 槽位释放 → 领第二个
|
|
||||||
gate.resolve();
|
|
||||||
await orch.drain();
|
|
||||||
assert.deepEqual(started, [t1.id, t2.id]);
|
|
||||||
assert.equal(store.getTask(t2.id)!.status, 'exec_review');
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('成功路径:状态流转 + setResult(四字段) + executor/reviewer/security 三 run succeeded', async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
|
||||||
const t = store.createTask({ projectId, title: 'tweak', complexity: 'easy' });
|
|
||||||
store.setOperations(t.id, '在 README.md 追加一行');
|
|
||||||
|
|
||||||
const seen: string[] = [];
|
|
||||||
store.subscribe((e) => { if (e.type === 'status.changed' && e.taskId === t.id) seen.push(String(e.payload.to)); });
|
|
||||||
|
|
||||||
let codeReportSeen: string | null = null;
|
|
||||||
let secReportSeen: string | null = null;
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
reviewCode: async (_task, _project, _wt, _runId, executorReport) => {
|
|
||||||
codeReportSeen = executorReport;
|
|
||||||
return okReview;
|
|
||||||
},
|
|
||||||
reviewSecurity: async (_task, _project, _wt, _runId, executorReport) => {
|
|
||||||
secReportSeen = executorReport;
|
|
||||||
return okSecurity;
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
|
|
||||||
const done = store.getTask(t.id)!;
|
|
||||||
assert.equal(done.status, 'exec_review');
|
|
||||||
assert.deepEqual(seen, ['queued', 'executing', 'exec_review']);
|
|
||||||
assert.deepEqual(done.result, {
|
|
||||||
branch: `maestro/${t.id}`,
|
|
||||||
worktree: `/tmp/fake-wt/${t.id}`,
|
|
||||||
diffSummary: ' README.md | 1 +',
|
|
||||||
commits: ['abc1234 hello maestro'],
|
|
||||||
prUrl: null,
|
|
||||||
summary: '## 做了什么\nmock 复审通过',
|
|
||||||
verdict: 'approve',
|
|
||||||
securitySummary: '## 安全审计\nmock 审计通过',
|
|
||||||
securityVerdict: 'approve',
|
|
||||||
mergeTaskId: null,
|
|
||||||
});
|
|
||||||
assert.equal(codeReportSeen, '执行自述:改了 README'); // runner finalText 传给两个复审作执行者自述
|
|
||||||
assert.equal(secReportSeen, '执行自述:改了 README');
|
|
||||||
|
|
||||||
const runs = store.listRuns(t.id);
|
|
||||||
assert.equal(runs.length, 3);
|
|
||||||
const executor = runs.find((r) => r.kind === 'executor')!;
|
|
||||||
assert.equal(executor.status, 'succeeded');
|
|
||||||
assert.equal(executor.branch, `maestro/${t.id}`);
|
|
||||||
assert.equal(executor.transcriptRef, '/tmp/fake.jsonl');
|
|
||||||
assert.equal(executor.claudeSessionId, 'sess-mock-1');
|
|
||||||
const reviewer = runs.find((r) => r.kind === 'reviewer')!;
|
|
||||||
assert.equal(reviewer.status, 'succeeded');
|
|
||||||
assert.equal(reviewer.transcriptRef, '/tmp/fake-review.jsonl');
|
|
||||||
assert.equal(reviewer.claudeSessionId, 'sess-review-1');
|
|
||||||
const security = runs.find((r) => r.kind === 'security')!;
|
|
||||||
assert.equal(security.status, 'succeeded');
|
|
||||||
assert.equal(security.transcriptRef, '/tmp/fake-security.jsonl');
|
|
||||||
assert.equal(security.claudeSessionId, 'sess-security-1');
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('任一 verdict=reject 也照常落进 result(最终裁决仍归用户)', async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
|
||||||
const t = store.createTask({ projectId, title: 'risky', complexity: 'easy' });
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
reviewCode: async () => ({ ...okReview, summary: '发现问题', verdict: 'reject' as const }),
|
|
||||||
reviewSecurity: async () => ({ ...okSecurity, summary: '发现密钥泄露', verdict: 'reject' as const }),
|
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
const done = store.getTask(t.id)!;
|
|
||||||
assert.equal(done.status, 'exec_review');
|
|
||||||
assert.equal(done.result!.verdict, 'reject');
|
|
||||||
assert.equal(done.result!.summary, '发现问题');
|
|
||||||
assert.equal(done.result!.securityVerdict, 'reject');
|
|
||||||
assert.equal(done.result!.securitySummary, '发现密钥泄露');
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('code review 失败不挡任务:summary 记失败原因、verdict=null,安全审计照常跑', async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
|
||||||
const t = store.createTask({ projectId, title: 'review-broken', complexity: 'easy' });
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
reviewCode: async () => { throw new Error('复审 CC 崩了'); },
|
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
|
|
||||||
const done = store.getTask(t.id)!;
|
|
||||||
assert.equal(done.status, 'exec_review'); // 不挡结果闸
|
|
||||||
assert.equal(done.result!.verdict, null);
|
|
||||||
assert.equal(done.result!.summary, '自动复审失败:复审 CC 崩了');
|
|
||||||
assert.equal(done.result!.securityVerdict, 'approve'); // 另一个复审不受影响
|
|
||||||
assert.equal(done.result!.securitySummary, '## 安全审计\nmock 审计通过');
|
|
||||||
|
|
||||||
const runs = store.listRuns(t.id);
|
|
||||||
assert.equal(runs.find((r) => r.kind === 'executor')!.status, 'succeeded');
|
|
||||||
const reviewer = runs.find((r) => r.kind === 'reviewer')!;
|
|
||||||
assert.equal(reviewer.status, 'failed');
|
|
||||||
assert.match(reviewer.error ?? '', /复审 CC 崩了/);
|
|
||||||
assert.equal(runs.find((r) => r.kind === 'security')!.status, 'succeeded');
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('安全审计失败不挡任务:securitySummary 记失败原因、securityVerdict=null,code review 不受影响', async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
|
||||||
const t = store.createTask({ projectId, title: 'security-broken', complexity: 'easy' });
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
reviewSecurity: async () => { throw new Error('审计 CC 崩了'); },
|
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
|
|
||||||
const done = store.getTask(t.id)!;
|
|
||||||
assert.equal(done.status, 'exec_review');
|
|
||||||
assert.equal(done.result!.verdict, 'approve');
|
|
||||||
assert.equal(done.result!.summary, '## 做了什么\nmock 复审通过');
|
|
||||||
assert.equal(done.result!.securityVerdict, null);
|
|
||||||
assert.equal(done.result!.securitySummary, '自动复审失败:审计 CC 崩了');
|
|
||||||
|
|
||||||
const runs = store.listRuns(t.id);
|
|
||||||
assert.equal(runs.find((r) => r.kind === 'executor')!.status, 'succeeded');
|
|
||||||
assert.equal(runs.find((r) => r.kind === 'reviewer')!.status, 'succeeded');
|
|
||||||
const security = runs.find((r) => r.kind === 'security')!;
|
|
||||||
assert.equal(security.status, 'failed');
|
|
||||||
assert.match(security.error ?? '', /审计 CC 崩了/);
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('verify 不过:按失败处理(run failed + 重新入队)', async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
|
||||||
const t = store.createTask({ projectId, title: 'v', complexity: 'easy' });
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
verify: async () => ({ ok: false, exitCode: 1, logRef: '/tmp/v.log', error: 'verify 失败(exit 1)' }),
|
|
||||||
}));
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'queued'); // 第一次失败 → 重新入队
|
|
||||||
const runs = store.listRuns(t.id);
|
|
||||||
assert.equal(runs.length, 1);
|
|
||||||
assert.equal(runs[0].status, 'failed');
|
|
||||||
assert.match(runs[0].error ?? '', /verify 失败/);
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test(`失败重试:重试 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention(共 ${DEFAULT_MAX_RETRIES + 1} 次失败 run)`, async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
|
||||||
const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });
|
|
||||||
|
|
||||||
let attempts = 0;
|
|
||||||
// 使用快进时钟跳过退避冷却(每次 tick 前将时钟推进 10min)
|
|
||||||
const clock = makeClock();
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: `boom #${attempts}` }; },
|
|
||||||
nowMs: clock.nowMs,
|
|
||||||
}));
|
|
||||||
|
|
||||||
for (let i = 1; i <= DEFAULT_MAX_RETRIES; i++) {
|
|
||||||
clock.advance(10 * 60_000); // 跳过退避冷却
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'queued', `第 ${i} 次失败后应重新入队`);
|
|
||||||
}
|
|
||||||
|
|
||||||
clock.advance(10 * 60_000); // 跳过最后一次退避
|
|
||||||
orch.tick(); // 最后一次重试也失败
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
|
|
||||||
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
|
|
||||||
|
|
||||||
const failed = store.listRuns(t.id).filter((r) => r.status === 'failed');
|
|
||||||
assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);
|
|
||||||
|
|
||||||
clock.advance(10 * 60_000);
|
|
||||||
orch.tick(); // needs_attention 不会再被领取
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('项目级 maxRetries=1:只重试 1 次就 → needs_attention', async () => {
|
|
||||||
const store = new Store(':memory:');
|
|
||||||
const p = store.createProject({
|
|
||||||
name: 'orch', repoPath: '/tmp/orch-repo-retries-' + Math.random(),
|
|
||||||
autonomy: 'auto-easy', maxRetries: 1,
|
|
||||||
});
|
|
||||||
const projectId = p.id;
|
|
||||||
const t = store.createTask({ projectId, title: 'fail1', complexity: 'easy' });
|
|
||||||
|
|
||||||
let attempts = 0;
|
|
||||||
const clock = makeClock();
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
|
|
||||||
nowMs: clock.nowMs,
|
|
||||||
}));
|
|
||||||
|
|
||||||
// 第 1 次执行失败 → 重新入队(还有 1 次重试机会)
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'queued');
|
|
||||||
|
|
||||||
// 第 2 次失败 → 超过 maxRetries=1 → needs_attention(快进时钟跳过退避)
|
|
||||||
clock.advance(10 * 60_000);
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
|
|
||||||
assert.equal(attempts, 2);
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('项目级 maxRetries=0:首次失败直接 needs_attention(不重试)', async () => {
|
|
||||||
const store = new Store(':memory:');
|
|
||||||
const p = store.createProject({
|
|
||||||
name: 'orch', repoPath: '/tmp/orch-repo-zero-' + Math.random(),
|
|
||||||
autonomy: 'auto-easy', maxRetries: 0,
|
|
||||||
});
|
|
||||||
const t = store.createTask({ projectId: p.id, title: 'fail0', complexity: 'easy' });
|
|
||||||
|
|
||||||
let attempts = 0;
|
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
|
||||||
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
|
|
||||||
}));
|
|
||||||
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
|
|
||||||
assert.equal(attempts, 1);
|
|
||||||
store.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('computeBackoffMs:指数退避公式,30s 基数,上限 10min', () => {
|
|
||||||
assert.equal(computeBackoffMs(1), 30_000);
|
|
||||||
assert.equal(computeBackoffMs(2), 60_000);
|
|
||||||
assert.equal(computeBackoffMs(3), 120_000);
|
|
||||||
assert.equal(computeBackoffMs(4), 240_000);
|
|
||||||
assert.equal(computeBackoffMs(10), 10 * 60_000); // capped at 10min
|
|
||||||
assert.equal(computeBackoffMs(100), 10 * 60_000);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('退避冷却:首次失败重入队后,退避期内不被领取;退避过期后正常领取', async () => {
|
|
||||||
const { store, projectId } = setup('auto-easy');
|
const { store, projectId } = setup('auto-easy');
|
||||||
const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
|
const t = store.createTask({ projectId, title: 'backoff', complexity: 'easy' });
|
||||||
|
// 任务已在 queued 且退避到未来
|
||||||
|
store.transition(t.id, 'queued', { by: 'test' });
|
||||||
|
const future = new Date(Date.now() + 60_000).toISOString();
|
||||||
|
store.setNextEligibleAt(t.id, future);
|
||||||
|
|
||||||
let attempts = 0;
|
const clock = makeClock(Date.now());
|
||||||
// 使用可控时钟:初始时间 0,退避 30s(attempt=1 → BACKOFF_BASE=30000ms)
|
const state = freshState();
|
||||||
const clock = makeClock(0);
|
const { deps } = mockDeps(state, { nowMs: clock.nowMs });
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
|
|
||||||
nowMs: clock.nowMs,
|
|
||||||
}));
|
|
||||||
|
|
||||||
// 第 1 次失败(t=0)→ 重入队 + 退避(backoffUntil = 0 + 30000 = 30000ms)
|
orch.claimTick();
|
||||||
orch.tick();
|
assert.equal(state.spawned.length, 0, '退避期内不领');
|
||||||
await orch.drain();
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'queued');
|
|
||||||
assert.equal(attempts, 1);
|
|
||||||
|
|
||||||
// t=29999:退避期内,不被领取
|
|
||||||
clock.advance(29_999);
|
|
||||||
orch.tick();
|
|
||||||
await orch.drain();
|
|
||||||
assert.equal(attempts, 1, '退避期内不应再次执行');
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'queued');
|
assert.equal(store.getTask(t.id)!.status, 'queued');
|
||||||
|
|
||||||
// t=30001:退避过期,任务应被重新领取
|
clock.advance(61_000); // 退避过期
|
||||||
clock.advance(2);
|
orch.claimTick();
|
||||||
orch.tick();
|
assert.equal(state.spawned.length, 1, '退避过期后领取');
|
||||||
await orch.drain();
|
assert.equal(store.getTask(t.id)!.status, 'executing');
|
||||||
assert.equal(attempts, 2, '退避过期后应再次执行');
|
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('requeueTask:needs_attention → queued,重置重试基线,再次允许重试', async () => {
|
test('spawnWorker 抛错:兜底 failTaskAttempt(task 转 queued,executor run failed)', () => {
|
||||||
const { store, projectId } = setup('auto-easy');
|
const { store, projectId } = setup('auto-easy');
|
||||||
const t = store.createTask({ projectId, title: 'requeue', complexity: 'easy' });
|
const t = store.createTask({ projectId, title: 'boom-spawn', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state, {
|
||||||
|
spawnWorker: () => { throw new Error('spawn 失败:ENOENT'); },
|
||||||
|
});
|
||||||
|
createOrchestrator(store, noopLog, deps).claimTick();
|
||||||
|
|
||||||
let attempts = 0;
|
const after = store.getTask(t.id)!;
|
||||||
const clock = makeClock();
|
assert.equal(after.status, 'queued', '默认 maxRetries=2,首次失败 → 重入队');
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
assert.ok(after.nextEligibleAt, '退避已持久化');
|
||||||
runner: async () => { attempts++; return { ok: false, transcriptRef: null, sessionId: null, error: 'boom' }; },
|
const executor = store.listRuns(t.id).find((r) => r.kind === 'executor')!;
|
||||||
nowMs: clock.nowMs,
|
assert.equal(executor.status, 'failed');
|
||||||
}));
|
assert.match(executor.error ?? '', /spawn 失败/);
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
// 耗尽默认重试次数 → needs_attention(每次需快进时钟跳过退避)
|
// ───────────────────────── reaper(回收死 worker)─────────────────────────
|
||||||
for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
|
|
||||||
clock.advance(10 * 60_000);
|
test('reaper:isWorkerAlive=false → failTaskAttempt(executing→queued,executor run failed)', () => {
|
||||||
orch.tick();
|
const { store, projectId } = setup('auto-easy');
|
||||||
await orch.drain();
|
const t = store.createTask({ projectId, title: 'dead', complexity: 'easy' });
|
||||||
}
|
// 先让它进 executing + 建 run(用一次正常领取,alive=true 不会被回收)
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
|
orch.claimTick();
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'executing');
|
||||||
|
const runId = store.listRuns(t.id)[0].id;
|
||||||
|
|
||||||
|
// 标记 worker 已死 → reaper 回收
|
||||||
|
state.alive = false;
|
||||||
|
orch.reap();
|
||||||
|
|
||||||
|
const after = store.getTask(t.id)!;
|
||||||
|
assert.equal(after.status, 'queued', '默认 maxRetries=2,首次回收 → 重入队');
|
||||||
|
assert.ok(after.nextEligibleAt, '退避已持久化');
|
||||||
|
const executor = store.getRun(runId)!;
|
||||||
|
assert.equal(executor.status, 'failed');
|
||||||
|
assert.match(executor.error ?? '', /worker 异常退出/);
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('reaper:isWorkerAlive=true → 不回收(保持 executing)', () => {
|
||||||
|
const { store, projectId } = setup('auto-easy');
|
||||||
|
const t = store.createTask({ projectId, title: 'alive', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
|
orch.claimTick();
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'executing');
|
||||||
|
|
||||||
|
state.alive = true;
|
||||||
|
orch.reap();
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'executing', 'worker 活 → 不回收');
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('reaper:maxRetries=0 → 死 worker 直接 needs_attention', () => {
|
||||||
|
const store = new Store(':memory:');
|
||||||
|
const p = store.createProject({
|
||||||
|
name: 'orch', repoPath: '/tmp/orch-reap0-' + Math.random(), autonomy: 'auto-easy', maxRetries: 0,
|
||||||
|
});
|
||||||
|
const t = store.createTask({ projectId: p.id, title: 'x', complexity: 'easy' });
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state);
|
||||||
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
|
orch.claimTick();
|
||||||
|
state.alive = false;
|
||||||
|
orch.reap();
|
||||||
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
|
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
|
||||||
assert.equal(attempts, DEFAULT_MAX_RETRIES + 1);
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
// 手动重投:重置基线,转 queued
|
// ───────────────────────── 多轮:reaper 腾槽后重领(退避到期)─────────────────────────
|
||||||
const requeued = store.requeueTask(t.id);
|
|
||||||
assert.equal(requeued.status, 'queued');
|
|
||||||
assert.equal(requeued.retryBaseline, DEFAULT_MAX_RETRIES + 1);
|
|
||||||
|
|
||||||
// 重投后编排器应能再次执行(快进时钟确保无退避阻拦)
|
test('死 worker 回收 + 退避到期后下一轮重新领取(监工闭环)', () => {
|
||||||
|
const { store, projectId } = setup('auto-easy', 1);
|
||||||
|
const t = store.createTask({ projectId, title: 'recycle', complexity: 'easy' });
|
||||||
|
const clock = makeClock(Date.now());
|
||||||
|
const state = freshState();
|
||||||
|
const { deps } = mockDeps(state, { nowMs: clock.nowMs });
|
||||||
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
|
|
||||||
|
// 第一轮:领取 + spawn(alive=true)
|
||||||
|
orch.tick();
|
||||||
|
assert.equal(state.spawned.length, 1);
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'executing');
|
||||||
|
|
||||||
|
// worker 死 → 下一轮 reaper 回收(→ queued + 退避),退避期内不重领
|
||||||
|
state.alive = false;
|
||||||
|
orch.tick();
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'queued');
|
||||||
|
assert.equal(state.spawned.length, 1, '退避期内不重领');
|
||||||
|
|
||||||
|
// 退避到期 → 再下一轮重新领取
|
||||||
clock.advance(10 * 60_000);
|
clock.advance(10 * 60_000);
|
||||||
orch.tick();
|
orch.tick();
|
||||||
await orch.drain();
|
assert.equal(state.spawned.length, 2, '退避到期 → 重新领取 spawn');
|
||||||
assert.equal(attempts, DEFAULT_MAX_RETRIES + 2, '重投后应再次执行');
|
assert.equal(store.getTask(t.id)!.status, 'executing');
|
||||||
// 第 1 次净失败后应重入队(基线已重置,净失败=1 < maxRetries=2)
|
|
||||||
assert.equal(store.getTask(t.id)!.status, 'queued');
|
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('project paused:不领取', async () => {
|
test(`reaper 反复回收:净失败 ${DEFAULT_MAX_RETRIES} 次后 → needs_attention`, () => {
|
||||||
const { store, projectId } = setup('auto-approved');
|
const { store, projectId } = setup('auto-easy', 1);
|
||||||
const t = store.createTask({ projectId, title: 'x', complexity: 'easy' });
|
const t = store.createTask({ projectId, title: 'flaky', complexity: 'easy' });
|
||||||
store.patchProject(projectId, { status: 'paused' });
|
const clock = makeClock(Date.now());
|
||||||
let calls = 0;
|
const state = freshState();
|
||||||
const orch = createOrchestrator(store, noopLog, mockDeps({
|
const { deps } = mockDeps(state, { nowMs: clock.nowMs });
|
||||||
runner: async () => { calls++; return okRun; },
|
const orch = createOrchestrator(store, noopLog, deps);
|
||||||
}));
|
|
||||||
orch.tick();
|
// 反复:领取(alive=true 领取那刻)→ 标死 → reaper 回收
|
||||||
await orch.drain();
|
for (let i = 0; i <= DEFAULT_MAX_RETRIES; i++) {
|
||||||
assert.equal(calls, 0);
|
clock.advance(10 * 60_000); // 跳过退避
|
||||||
assert.equal(store.getTask(t.id)!.status, 'ready');
|
state.alive = true;
|
||||||
|
orch.claimTick();
|
||||||
|
state.alive = false;
|
||||||
|
orch.reap();
|
||||||
|
}
|
||||||
|
assert.equal(store.getTask(t.id)!.status, 'needs_attention');
|
||||||
|
const failed = store.listRuns(t.id).filter((r) => r.kind === 'executor' && r.status === 'failed');
|
||||||
|
assert.equal(failed.length, DEFAULT_MAX_RETRIES + 1);
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user