feat(cleanup): exec_review 合并后回收 runs/<runId>/ 工作目录 + 定期扫尾 [tsk_6uziW1Dg-ftQ]

每个 run 在 ~/.maestro/runs/<runId>/ 留下 job.json/outbox.ndjson/heartbeat,
合并(exec_review accept)后不再需要,需回收避免无限堆积。

- protocol.ts:新增 removeRunDir / isPlainRunId(幂等删 runDir,runId 白名单防越界)
- cleanup.ts(新):
  - cleanupTaskRunArtifacts:合并/接受后即时回收某任务全部 run 工作目录
  - sweepRunArtifacts:周期扫尾终态/孤儿 runDir 与(可选)转录,按保留天数
  - isRunCleanable:仅终态(done/cancelled)/孤儿可回收,活跃/在途一律保留
  - 双保险:任务状态判定 + heartbeat 新鲜则跳过,绝不误删活跃 run
  - 保留策略可配:MAESTRO_RUN_RETENTION_DAYS(默认3) /
    MAESTRO_TRANSCRIPT_RETENTION_DAYS(默认0=永久保留,便于排查)
- server.ts:exec_review accept 合并成功后即时清 runDir(与 removeWorktree 同处)
- index.ts:startCleanupLoop 定时扫尾(MAESTRO_CLEANUP_INTERVAL 秒,默认3600,0=关闭)
- test/cleanup.test.ts:终态清/在途留/孤儿清/保留期/heartbeat 兜底/转录开关/配置

测试:npm test → 164 passed;npm run typecheck → clean

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-13 15:02:23 +08:00
parent f020e15137
commit a365aaf06a
5 changed files with 396 additions and 1 deletions
+5
View File
@@ -10,6 +10,7 @@ import { resolvedExecutorModels } from '../executor/models.js';
import { classifyComplexity, type ClassifierFn } from '../executor/classify.js';
import { mergeBranch } from '../executor/merge.js';
import { git, removeWorktree } from '../executor/worktree.js';
import { cleanupTaskRunArtifacts } from '../executor/cleanup.js';
import { resolveLogo, LOGO_MIME } from './logo.js';
import { readTranscript, TranscriptError } from '../executor/transcript.js';
import { createReadStream } from 'node:fs';
@@ -332,6 +333,10 @@ export function buildServer(opts: ApiOptions): FastifyInstance {
store.decide(id, 'accept', b.actor ?? 'user', b.reason ?? null);
// 合并产物记录:复用 prUrl 字段写 merged:<mergeCommit>
store.setResult(id, { ...task.result, prUrl: `merged:${mr.mergeCommit}` });
// 即时回收 runs/<runId>/job/outbox/heartbeat 已无价值;转录保留供排查)。
// 任务此刻已 done(终态),cleanupTaskRunArtifacts 内部据此放行;同步删(纯本地 fs,快)。
const reclaimed = cleanupTaskRunArtifacts(store, id, app.log);
if (reclaimed) app.log.info(`任务 ${id} 合并后回收 ${reclaimed} 个 run 工作目录`);
// 异步回收:执行 worktree + 已合并的任务分支(失败只记日志,不影响响应)
void (async () => {
try {
+33
View File
@@ -7,6 +7,7 @@ import { loadConfig } from './config.js';
import { startOrchestrator } from './orchestrator.js';
import { startNotifier } from './notify.js';
import { isWorkerAlive, heartbeatAgeMs } from '../executor/protocol.js';
import { sweepRunArtifacts, loadCleanupConfig } from '../executor/cleanup.js';
import { syncProject, todoJsonPath } from '../sync/todo-sync.js';
/**
@@ -49,6 +50,36 @@ function startSyncLoop(store: Store, app: FastifyInstance): NodeJS.Timeout | nul
return timer;
}
/**
* 定时回收 run 产物:周期扫尾 runs/<runId>/ 与转录(终态/孤儿、过保留期者)。
* 间隔由 MAESTRO_CLEANUP_INTERVAL(秒)控制,默认 36000=关闭;保留策略见 loadCleanupConfig。
* 错误只记日志,不崩 daemon。
*/
function startCleanupLoop(store: Store, app: FastifyInstance): NodeJS.Timeout | null {
const intervalSec = Number(process.env.MAESTRO_CLEANUP_INTERVAL ?? 3600);
if (!Number.isFinite(intervalSec) || intervalSec <= 0) {
app.log.info('定时 run 产物回收已关闭(MAESTRO_CLEANUP_INTERVAL=0');
return null;
}
const cfg = loadCleanupConfig();
const tick = (): void => {
try {
const r = sweepRunArtifacts(store, cfg, app.log);
if (r.runDirs || r.transcripts) {
app.log.info(`run 产物回收:runDir=${r.runDirs} 转录=${r.transcripts}`);
}
} catch (e) {
app.log.error(`run 产物回收轮失败:${(e as Error).message}`);
}
};
const timer = setInterval(tick, intervalSec * 1000);
timer.unref();
app.log.info(
`定时 run 产物回收已启用:每 ${intervalSec}s 一轮(runDir 保留 ${cfg.runRetentionDays}d,转录保留 ${cfg.transcriptRetentionDays > 0 ? cfg.transcriptRetentionDays + 'd' : '永久'}`,
);
return timer;
}
/** maestrod:核心 daemon。Store + REST/WS API + 定时同步 + 编排器(Phase 2:自动领取可执行任务并在 worktree 起 headless CC)。 */
async function main(): Promise<void> {
const cfg = loadConfig();
@@ -75,12 +106,14 @@ async function main(): Promise<void> {
const syncTimer = startSyncLoop(store, app);
const orchTimer = startOrchestrator(store, app); // 编排器:自动领取可执行任务(MAESTRO_ORCH_INTERVAL 秒,0=关闭)
const cleanupTimer = startCleanupLoop(store, app); // 定时回收 run 产物(MAESTRO_CLEANUP_INTERVAL 秒,0=关闭)
const stopNotifier = startNotifier(store, app); // macOS 原生通知:审核闸/需人工/合并完成(MAESTRO_NOTIFY=0 关闭)
const shutdown = async (): Promise<void> => {
app.log.info('收到退出信号,关闭中…');
if (syncTimer) clearInterval(syncTimer);
if (orchTimer) clearInterval(orchTimer);
if (cleanupTimer) clearInterval(cleanupTimer);
if (stopNotifier) stopNotifier();
await app.close();
store.close();
+166
View File
@@ -0,0 +1,166 @@
// Run 产物回收(Phase 3 多进程执行的扫尾)。
//
// 每个 run 在两处留下产物:
// - runs/<runId>/ ── job.json / outbox.ndjson / heartbeat(纯运行期脚手架,落库后无价值)
// - <transcriptDir>/<runId>.jsonl ── 转录(事后排查有价值,默认长保留)
//
// 终态任务(done/cancelled)的产物不再需要;orphan(任务已删,run 行被 FK 级联清掉)的产物同理。
// 本模块提供:
// - cleanupTaskRunArtifacts:合并/接受后即时回收某任务的全部 run 工作目录(server 调)。
// - sweepRunArtifacts :周期扫尾,按保留天数回收终态/孤儿 runDir 与(可选)转录(daemon 调)。
//
// 安全铁律:只删【终态或孤儿】run 的产物;活跃/在途/exec_review 的一律保留。
// 双保险:① 任务状态判定 ② heartbeat 新鲜则跳过(兜底防误删仍在写的 run)。
import { existsSync, readdirSync, rmSync, statSync } from 'node:fs';
import { join } from 'node:path';
import { Store } from '../store/index.js';
import { TERMINAL_STATUS } from '../model/status.js';
import {
runsBase, runDir, removeRunDir, isPlainRunId, heartbeatAgeMs, HEARTBEAT_GRACE_MS,
} from './protocol.js';
import { transcriptDir } from './cc.js';
const DAY_MS = 24 * 60 * 60 * 1000;
export interface CleanupLogger {
info(msg: string): void;
error(msg: string): void;
}
export interface CleanupConfig {
/** runs/<runId>/ 保留天数(自目录 mtime 起算);0 = 终态后立即可回收。 */
runRetentionDays: number;
/** <transcriptDir>/<runId>.jsonl 保留天数;<=0 = 永不自动清理(默认,便于事后排查)。 */
transcriptRetentionDays: number;
}
export const DEFAULT_CLEANUP_CONFIG: CleanupConfig = {
runRetentionDays: 3,
transcriptRetentionDays: 0,
};
/** 从环境变量读保留策略(覆盖默认)。非法值回退默认。 */
export function loadCleanupConfig(env: NodeJS.ProcessEnv = process.env): CleanupConfig {
const num = (raw: string | undefined, fallback: number): number => {
if (raw === undefined || raw.trim() === '') return fallback;
const n = Number(raw);
return Number.isFinite(n) && n >= 0 ? n : fallback;
};
return {
runRetentionDays: num(env.MAESTRO_RUN_RETENTION_DAYS, DEFAULT_CLEANUP_CONFIG.runRetentionDays),
transcriptRetentionDays: num(env.MAESTRO_TRANSCRIPT_RETENTION_DAYS, DEFAULT_CLEANUP_CONFIG.transcriptRetentionDays),
};
}
/**
* 该 run 的工作目录是否可回收:终态任务 / orphan(run 行或任务已不在 DB)→ 可;活跃/在途 → 否。
* 不看保留天数与 heartbeat(那是调用方叠加的策略/兜底)。
*/
export function isRunCleanable(store: Store, runId: string): boolean {
const run = store.getRun(runId);
if (!run) return true; // orphan:任务已删,run 行被 FK 级联清除 → 回收
const task = store.getTask(run.taskId);
if (!task) return true; // 同上(防御)
return TERMINAL_STATUS.has(task.status); // done / cancelled
}
/** heartbeat 仍新鲜(有进程在写)→ 不可删,兜底防误删活跃 run。 */
function heartbeatFresh(runId: string, now: number): boolean {
const age = heartbeatAgeMs(runId, now);
return age !== null && age < HEARTBEAT_GRACE_MS;
}
/**
* 即时回收某任务全部 run 的工作目录(runs/<runId>/)。合并成功 / 接受后由 server 调。
* 只删工作目录,不动转录(保留排查价值)。失败只计日志,绝不抛(不阻断主流程)。
* 返回实际清理的目录数。
*/
export function cleanupTaskRunArtifacts(store: Store, taskId: string, log: CleanupLogger): number {
let n = 0;
try {
for (const run of store.listRuns(taskId)) {
if (heartbeatFresh(run.id, Date.now())) continue; // 极少见:仍有进程在写则跳过
try {
removeRunDir(run.id);
n++;
} catch (e) {
log.error(`回收 run 目录 ${run.id} 失败(不影响结果):${(e as Error).message}`);
}
}
} catch (e) {
log.error(`回收任务 ${taskId} 的 run 产物失败(不影响结果):${(e as Error).message}`);
}
return n;
}
export interface SweepResult {
runDirs: number; // 清理的 runs/<runId>/ 目录数
transcripts: number; // 清理的转录文件数
}
/**
* 周期扫尾:遍历 runs/ 与 transcripts/,回收终态/孤儿且超过保留期的产物。
* - runDir:任务终态/孤儿 且 目录 mtime 早于 runRetentionDays 且 heartbeat 不新鲜 → 删。
* - 转录:transcriptRetentionDays>0 时,任务终态/孤儿 且 文件 mtime 早于该天数 → 删(活跃任务永不删)。
* 单条失败不阻断其余。返回清理计数。
*/
export function sweepRunArtifacts(
store: Store,
cfg: CleanupConfig,
log: CleanupLogger,
now: number = Date.now(),
): SweepResult {
const res: SweepResult = { runDirs: 0, transcripts: 0 };
// ── runs/<runId>/ ──
const base = runsBase();
if (existsSync(base)) {
let entries: string[] = [];
try { entries = readdirSync(base); } catch (e) { log.error(`读取 runs 目录失败:${(e as Error).message}`); }
const cutoff = now - cfg.runRetentionDays * DAY_MS;
for (const runId of entries) {
if (!isPlainRunId(runId)) continue;
const dir = runDir(runId);
try {
const st = statSync(dir);
if (!st.isDirectory()) continue;
if (!isRunCleanable(store, runId)) continue; // 活跃/在途 → 保留
if (st.mtimeMs > cutoff) continue; // 未过保留期
if (heartbeatFresh(runId, now)) continue; // 仍在写 → 兜底保留
removeRunDir(runId);
res.runDirs++;
} catch (e) {
log.error(`扫尾 run 目录 ${runId} 失败:${(e as Error).message}`);
}
}
}
// ── <transcriptDir>/<runId>.jsonl ──(默认关闭)
if (cfg.transcriptRetentionDays > 0) {
const tdir = transcriptDir();
if (existsSync(tdir)) {
let files: string[] = [];
try { files = readdirSync(tdir); } catch (e) { log.error(`读取转录目录失败:${(e as Error).message}`); }
const cutoff = now - cfg.transcriptRetentionDays * DAY_MS;
for (const file of files) {
if (!file.endsWith('.jsonl')) continue;
const runId = file.slice(0, -'.jsonl'.length);
if (!isPlainRunId(runId)) continue;
const fp = join(tdir, file);
try {
const st = statSync(fp);
if (!st.isFile()) continue;
if (!isRunCleanable(store, runId)) continue; // 活跃任务的转录 → 永不删
if (st.mtimeMs > cutoff) continue; // 未过保留期
rmSync(fp, { force: true });
res.transcripts++;
} catch (e) {
log.error(`扫尾转录 ${file} 失败:${(e as Error).message}`);
}
}
}
}
return res;
}
+16 -1
View File
@@ -10,7 +10,7 @@
import {
existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync,
statSync, utimesSync, closeSync, openSync,
statSync, utimesSync, closeSync, openSync, rmSync,
} from 'node:fs';
import { homedir } from 'node:os';
import { join } from 'node:path';
@@ -33,6 +33,21 @@ export function jobPath(runId: string): string { return join(runDir(runId), 'job
export function outboxPath(runId: string): string { return join(runDir(runId), 'outbox.ndjson'); }
export function heartbeatPath(runId: string): string { return join(runDir(runId), 'heartbeat'); }
/**
* 删除单个 run 的工作目录 runs/<runId>/job.json + outbox.ndjson + heartbeat)。
* 幂等(目录不存在不报错)。合并/终态后由 daemon 侧回收调用——worker 自身从不删自己的工作目录。
* 仅接受形如 run_xxx 的纯 id(无路径分隔符),避免越界删除。
*/
export function removeRunDir(runId: string): void {
if (!isPlainRunId(runId)) return;
rmSync(runDir(runId), { recursive: true, force: true });
}
/** runId 合法性:仅字母数字与 -_,杜绝 . / 等可越界字符(清理路径计算的安全前提)。 */
export function isPlainRunId(runId: string): boolean {
return /^[A-Za-z0-9_-]+$/.test(runId);
}
// ───────────────────────── daemon → workerJobSpec ─────────────────────────
/**
+176
View File
@@ -0,0 +1,176 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync, utimesSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Store } from '../src/store/index.js';
import { writeJobSpec, runDir, touchHeartbeat, type JobSpec } from '../src/executor/protocol.js';
import { transcriptDir } from '../src/executor/cc.js';
import {
cleanupTaskRunArtifacts, sweepRunArtifacts, isRunCleanable, loadCleanupConfig,
type CleanupConfig,
} from '../src/executor/cleanup.js';
import type { Project, Task } from '../src/model/types.js';
const noopLog = { info: (): void => undefined, error: (): void => undefined };
const DAY_MS = 24 * 60 * 60 * 1000;
/** 隔离 MAESTRO_DATA_DIRrunDir/transcriptDir 调用时读 env),跑回调后清理。 */
function withTmpDataDir(cb: (dataDir: string) => void): void {
const dir = mkdtempSync(join(tmpdir(), 'maestro-cleanup-'));
const prev = process.env.MAESTRO_DATA_DIR;
process.env.MAESTRO_DATA_DIR = dir;
try {
cb(dir);
} finally {
if (prev === undefined) delete process.env.MAESTRO_DATA_DIR;
else process.env.MAESTRO_DATA_DIR = prev;
rmSync(dir, { recursive: true, force: true });
}
}
/** 建项目 + executor run,落 job.json/outbox/heartbeat 到磁盘;返回 store/ids。 */
function setupRun(): { store: Store; taskId: string; runId: string } {
const store = new Store(':memory:');
const p = store.createProject({ name: 'cl', repoPath: '/tmp/cl-' + Math.random(), autonomy: 'auto-easy' });
const t = store.createTask({ projectId: p.id, title: 'tweak', complexity: 'easy' });
store.setOperations(t.id, 'x');
store.transition(t.id, 'queued', { by: 'test' });
store.transition(t.id, 'executing', { by: 'test' });
const run = store.startRun(t.id, 'executor', { worktree: '/tmp/wt', branch: 'b' });
// 落产物
const job: JobSpec = {
runId: run.id,
task: t as Task,
project: p as Project,
worktreeDir: '/tmp/wt',
branch: 'b',
};
writeJobSpec(job);
return { store, taskId: t.id, runId: run.id };
}
/** 把 runDir 的 mtime 拨老 days 天(绕过保留期)。 */
function ageDir(runId: string, days: number): void {
const when = new Date(Date.now() - days * DAY_MS);
utimesSync(runDir(runId), when, when);
}
test('isRunCleanable:终态/孤儿可回收,在途不可', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupRun();
assert.equal(isRunCleanable(store, runId), false); // executing → 否
store.transition(taskId, 'exec_review', { by: 'test' });
assert.equal(isRunCleanable(store, runId), false); // exec_review → 否
store.transition(taskId, 'done', { by: 'test' });
assert.equal(isRunCleanable(store, runId), true); // done → 是
assert.equal(isRunCleanable(store, 'run_does_not_exist'), true); // 孤儿 → 是
});
});
test('cleanupTaskRunArtifacts:终态任务的 runDir 被即时清,转录保留', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupRun();
// 造一份转录
mkdirSync(transcriptDir(), { recursive: true });
const tref = join(transcriptDir(), `${runId}.jsonl`);
writeFileSync(tref, '{}\n');
store.transition(taskId, 'exec_review', { by: 'test' });
store.transition(taskId, 'done', { by: 'test' });
assert.ok(existsSync(runDir(runId)));
const n = cleanupTaskRunArtifacts(store, taskId, noopLog);
assert.equal(n, 1);
assert.equal(existsSync(runDir(runId)), false); // runDir 清
assert.ok(existsSync(tref)); // 转录保留
});
});
test('cleanupTaskRunArtifactsheartbeat 新鲜则跳过(不误删活跃 run)', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupRun();
touchHeartbeat(runId); // 新鲜心跳
store.transition(taskId, 'exec_review', { by: 'test' });
store.transition(taskId, 'done', { by: 'test' });
const n = cleanupTaskRunArtifacts(store, taskId, noopLog);
assert.equal(n, 0);
assert.ok(existsSync(runDir(runId)));
});
});
const cfg: CleanupConfig = { runRetentionDays: 1, transcriptRetentionDays: 0 };
test('sweepRunArtifactsdone 任务过保留期 → 清;在途 → 留', () => {
withTmpDataDir(() => {
const a = setupRun();
const store = a.store;
// 第二个 run(同 store):保持 executing
const t2 = store.createTask({ projectId: store.listProjects()[0].id, title: 't2', complexity: 'easy' });
store.setOperations(t2.id, 'y');
store.transition(t2.id, 'queued', { by: 'test' });
store.transition(t2.id, 'executing', { by: 'test' });
const run2 = store.startRun(t2.id, 'executor', { worktree: '/tmp/wt2', branch: 'b2' });
writeJobSpec({ runId: run2.id, task: t2 as Task, project: store.listProjects()[0] as Project, worktreeDir: '/tmp/wt2', branch: 'b2' });
// a → done 且拨老;run2 留在 executing
store.transition(a.taskId, 'exec_review', { by: 'test' });
store.transition(a.taskId, 'done', { by: 'test' });
ageDir(a.runId, 2);
ageDir(run2.id, 2); // 即便老,executing 也不该删
const res = sweepRunArtifacts(store, cfg, noopLog);
assert.equal(res.runDirs, 1);
assert.equal(existsSync(runDir(a.runId)), false); // done → 清
assert.ok(existsSync(runDir(run2.id))); // executing → 留
});
});
test('sweepRunArtifacts:未过保留期不清;孤儿过期清', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupRun();
store.transition(taskId, 'cancelled', { by: 'test' });
// 刚创建(mtime≈now),未过 1d 保留期 → 不清
let res = sweepRunArtifacts(store, cfg, noopLog);
assert.equal(res.runDirs, 0);
assert.ok(existsSync(runDir(runId)));
// 拨老 → 清
ageDir(runId, 2);
res = sweepRunArtifacts(store, cfg, noopLog);
assert.equal(res.runDirs, 1);
assert.equal(existsSync(runDir(runId)), false);
});
});
test('sweepRunArtifacts:转录开关(>0 时清终态过期,活跃永留)', () => {
withTmpDataDir(() => {
const { store, taskId, runId } = setupRun();
mkdirSync(transcriptDir(), { recursive: true });
const tref = join(transcriptDir(), `${runId}.jsonl`);
writeFileSync(tref, '{}\n');
const old = new Date(Date.now() - 5 * DAY_MS);
utimesSync(tref, old, old);
// 活跃(executing):转录永不删,即便过期
let res = sweepRunArtifacts(store, { runRetentionDays: 1, transcriptRetentionDays: 2 }, noopLog);
assert.equal(res.transcripts, 0);
assert.ok(existsSync(tref));
// 转为终态 → 过期转录被清
store.transition(taskId, 'cancelled', { by: 'test' });
res = sweepRunArtifacts(store, { runRetentionDays: 1, transcriptRetentionDays: 2 }, noopLog);
assert.equal(res.transcripts, 1);
assert.equal(existsSync(tref), false);
});
});
test('loadCleanupConfigenv 覆盖与非法回退', () => {
const base = loadCleanupConfig({} as NodeJS.ProcessEnv);
assert.equal(base.runRetentionDays, 3);
assert.equal(base.transcriptRetentionDays, 0);
const over = loadCleanupConfig({ MAESTRO_RUN_RETENTION_DAYS: '7', MAESTRO_TRANSCRIPT_RETENTION_DAYS: '30' } as unknown as NodeJS.ProcessEnv);
assert.equal(over.runRetentionDays, 7);
assert.equal(over.transcriptRetentionDays, 30);
const bad = loadCleanupConfig({ MAESTRO_RUN_RETENTION_DAYS: 'abc' } as unknown as NodeJS.ProcessEnv);
assert.equal(bad.runRetentionDays, 3);
});