phase3(契约层): protocol.ts + 落盘状态 + store 失败策略/判活原语
多进程执行解耦的 Phase 0 共享契约（worker 与 daemon 都 import 它）： - executor/protocol.ts: JobSpec(daemon→worker 唯一输入) + OutboxRecord(worker→daemon) + 文件 IO(job/outbox/heartbeat) + isWorkerAlive(心跳为主/boot 窗口 pid 兜底/防 pid 复用) - schema+db+types+mappers: runs.worker_pid/last_seq、tasks.next_eligible_at(持久化退避) - store: failTaskAttempt(失败/重试策略唯一落点) + setNextEligibleAt/setWorkerPid/setLastSeq + countExecuting + executingWithLatestExecutorRun + reconcileInterrupted(isAlive 注入,默认保守回收) 纯增量,140 测试全绿。worker.ts/pipeline.ts(worker 侧) 与 orchestrator/ingest/index(daemon 侧) 待接。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,162 @@
|
||||
// Worker ↔ daemon 契约（Phase 3 多进程执行）。
|
||||
//
|
||||
// 铁律：worker 进程【完全不碰 DB】（连读都不碰）。daemon 是唯一 DB 读写者。
|
||||
// 二者只经【文件 + 进程信号】通讯，天然跨 daemon 重启持久：
|
||||
// daemon → worker：runs/<runId>/job.json（唯一输入）；SIGTERM（取消/超时）
|
||||
// worker → daemon：runs/<runId>/outbox.ndjson（追加、带 seq）；runs/<runId>/heartbeat（mtime 心跳）
|
||||
//
|
||||
// 本模块同时被 worker 侧（写 outbox/读 job/刷心跳）与 daemon 侧（写 job/读 outbox/判活）import，
|
||||
// 是唯一的共享面——双方都不应另起自己的协议常量/路径计算。
|
||||
|
||||
import {
|
||||
existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync,
|
||||
statSync, utimesSync, closeSync, openSync,
|
||||
} from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { Project, Task, ReviewVerdict } from '../model/types.js';
|
||||
|
||||
/** 数据根目录（与 worktree.ts 同约定）：<MAESTRO_DATA_DIR 或 ~/.maestro> */
|
||||
function dataDir(): string {
|
||||
return process.env.MAESTRO_DATA_DIR ?? join(homedir(), '.maestro');
|
||||
}
|
||||
|
||||
/** 全部 run 工作目录根：<data>/runs */
|
||||
export function runsBase(): string {
|
||||
return join(dataDir(), 'runs');
|
||||
}
|
||||
/** 单个 run 的工作目录：<runsBase>/<runId>（放 job.json / outbox.ndjson / heartbeat） */
|
||||
export function runDir(runId: string): string {
|
||||
return join(runsBase(), runId);
|
||||
}
|
||||
export function jobPath(runId: string): string { return join(runDir(runId), 'job.json'); }
|
||||
export function outboxPath(runId: string): string { return join(runDir(runId), 'outbox.ndjson'); }
|
||||
export function heartbeatPath(runId: string): string { return join(runDir(runId), 'heartbeat'); }
|
||||
|
||||
// ───────────────────────── daemon → worker：JobSpec ─────────────────────────
|
||||
|
||||
/**
|
||||
* worker 的唯一输入。daemon 在 spawn 前写 runs/<runId>/job.json，worker 读它即可执行，
|
||||
* 【无需访问 DB】。携带完整 Task / Project（均 JSON 可序列化），worker 自行 pickModel/buildPrompt。
|
||||
*/
|
||||
export interface JobSpec {
|
||||
runId: string;
|
||||
task: Task;
|
||||
project: Project;
|
||||
/** 确定性 worktree 路径与分支（daemon 预填；worker 也能由 worktree.ts 算出，传入避免重复） */
|
||||
worktreeDir: string;
|
||||
branch: string;
|
||||
}
|
||||
|
||||
export function writeJobSpec(job: JobSpec): void {
|
||||
mkdirSync(runDir(job.runId), { recursive: true });
|
||||
writeFileSync(jobPath(job.runId), JSON.stringify(job));
|
||||
}
|
||||
export function readJobSpec(runId: string): JobSpec {
|
||||
return JSON.parse(readFileSync(jobPath(runId), 'utf8')) as JobSpec;
|
||||
}
|
||||
|
||||
// ───────────────────────── worker → daemon：OutboxRecord ────────────────────
|
||||
|
||||
/** 一次复审（code review / 安全审计）的产出，worker 报给 daemon，由 daemon 落成 run 行 + result 字段 */
|
||||
export interface ReviewReport {
|
||||
summary: string | null;
|
||||
verdict: ReviewVerdict | null;
|
||||
transcriptRef: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* worker 追加进 outbox 的记录。`seq` 单调递增（每 run 内），`at` ISO 时间，均由 appendOutbox 填。
|
||||
* - started：worker 启动，报自己的 pid / worktree / branch / model
|
||||
* - phase ：进度阶段（executing|verifying|reviewing…），daemon 落成 event 推看板（可选）
|
||||
* - failed ：本次尝试终态失败（executor 报错或 verify 失败）。daemon 据此 finishRun(failed)+重试决策
|
||||
* - result ：成功终态。daemon 据此建 reviewer/security run + setResult(四字段) + 转 exec_review
|
||||
* - done ：worker 即将退出（成功或失败都发，daemon 据此停止 tail）
|
||||
*/
|
||||
export type OutboxRecord =
|
||||
| { seq: number; at: string; type: 'started'; pid: number; worktree: string; branch: string; model: string }
|
||||
| { seq: number; at: string; type: 'phase'; phase: string }
|
||||
| { seq: number; at: string; type: 'failed'; error: string; transcriptRef: string | null; sessionId: string | null }
|
||||
| {
|
||||
seq: number; at: string; type: 'result';
|
||||
branch: string; worktree: string; diffSummary: string; commits: string[];
|
||||
executor: { transcriptRef: string | null; sessionId: string | null };
|
||||
code: ReviewReport;
|
||||
security: ReviewReport;
|
||||
}
|
||||
| { seq: number; at: string; type: 'done' };
|
||||
|
||||
/** OutboxRecord 去掉 seq/at（由 appendOutbox 填） */
|
||||
export type OutboxPayload =
|
||||
| Omit<Extract<OutboxRecord, { type: 'started' }>, 'seq' | 'at'>
|
||||
| Omit<Extract<OutboxRecord, { type: 'phase' }>, 'seq' | 'at'>
|
||||
| Omit<Extract<OutboxRecord, { type: 'failed' }>, 'seq' | 'at'>
|
||||
| Omit<Extract<OutboxRecord, { type: 'result' }>, 'seq' | 'at'>
|
||||
| Omit<Extract<OutboxRecord, { type: 'done' }>, 'seq' | 'at'>;
|
||||
|
||||
/** 追加一条 outbox 记录（worker 侧调用）。seq = 现有行数+1（worker 单线程，无并发写）。返回写入的完整记录。 */
|
||||
export function appendOutbox(runId: string, payload: OutboxPayload): OutboxRecord {
|
||||
mkdirSync(runDir(runId), { recursive: true });
|
||||
const prev = readOutboxAll(runId);
|
||||
const seq = prev.length + 1;
|
||||
const rec = { seq, at: new Date().toISOString(), ...payload } as OutboxRecord;
|
||||
appendFileSync(outboxPath(runId), JSON.stringify(rec) + '\n');
|
||||
return rec;
|
||||
}
|
||||
|
||||
/** 读全部 outbox 记录（坏行跳过，容忍写一半的尾行）。 */
|
||||
export function readOutboxAll(runId: string): OutboxRecord[] {
|
||||
const p = outboxPath(runId);
|
||||
if (!existsSync(p)) return [];
|
||||
const out: OutboxRecord[] = [];
|
||||
for (const line of readFileSync(p, 'utf8').split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try { out.push(JSON.parse(s) as OutboxRecord); } catch { /* 半截尾行：跳过，下轮再读 */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 读 seq > lastSeq 的新记录（daemon ingest 侧调用，幂等去重靠 lastSeq）。 */
|
||||
export function readOutboxSince(runId: string, lastSeq: number): OutboxRecord[] {
|
||||
return readOutboxAll(runId).filter((r) => r.seq > lastSeq);
|
||||
}
|
||||
|
||||
// ───────────────────────── worker → daemon：心跳 + 存活判定 ──────────────────
|
||||
|
||||
/** worker 刷心跳：touch runs/<runId>/heartbeat（文件不存在则建）。daemon 用其 mtime 判活。 */
|
||||
export function touchHeartbeat(runId: string): void {
|
||||
mkdirSync(runDir(runId), { recursive: true });
|
||||
const p = heartbeatPath(runId);
|
||||
if (!existsSync(p)) { closeSync(openSync(p, 'w')); return; }
|
||||
const now = new Date();
|
||||
utimesSync(p, now, now);
|
||||
}
|
||||
|
||||
/** 心跳文件的「年龄」毫秒（now - mtime）；无心跳文件返回 null。 */
|
||||
export function heartbeatAgeMs(runId: string, now = Date.now()): number | null {
|
||||
try { return now - statSync(heartbeatPath(runId)).mtimeMs; }
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
/** pid 是否存活（signal 0 探测；EPERM 视为存活——进程在但无权） */
|
||||
export function pidAlive(pid: number): boolean {
|
||||
if (!pid || pid <= 0) return false;
|
||||
try { process.kill(pid, 0); return true; }
|
||||
catch (e) { return (e as NodeJS.ErrnoException).code === 'EPERM'; }
|
||||
}
|
||||
|
||||
export const HEARTBEAT_INTERVAL_MS = 10_000; // worker 刷心跳间隔
|
||||
export const HEARTBEAT_GRACE_MS = 60_000; // 心跳超过此年龄视为可疑（须 > interval，容忍 GC/慢盘）
|
||||
export const BOOT_GRACE_MS = 30_000; // started 后此窗口内额外认 pid 存活（覆盖启动到首次心跳空窗）
|
||||
|
||||
/**
|
||||
* worker 是否存活。主信号＝心跳新鲜；boot 窗口内额外接受 pid 存活（覆盖启动空窗）；
|
||||
* 窗口后仅认心跳——规避 pid 复用误判（被复用的 pid 不会更新本 run 的心跳）。
|
||||
*/
|
||||
export function isWorkerAlive(args: { pid: number | null; heartbeatAgeMs: number | null; startedAgeMs: number }): boolean {
|
||||
const { pid, heartbeatAgeMs: hb, startedAgeMs } = args;
|
||||
if (hb !== null && hb < HEARTBEAT_GRACE_MS) return true; // 心跳新鲜 → 活
|
||||
if (startedAgeMs < BOOT_GRACE_MS && pid !== null && pidAlive(pid)) return true; // 启动空窗 → pid 兜底
|
||||
return false;
|
||||
}
|
||||
Reference in New Issue
Block a user