maestro/test/metrics.test.ts

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { Store } from '../src/store/index.js';
import {
  percentile, ratio, estimateCostUnits, emptyMetrics, costPerMin, DEFAULT_COST_PER_MIN,
} from '../src/model/metrics.js';
import type { RunStatus } from '../src/model/types.js';

function fresh(): Store { return new Store(':memory:'); }

const BASE = 1_700_000_000_000;   // 固定基准时刻，避免依赖真实时间

/** 造一条 executor run 并精确设置起止时间（durMs=null → 未结束）。 */
function mkRun(
  s: Store, taskId: string,
  opts: { status?: RunStatus; startMs?: number; durMs?: number | null } = {},
): string {
  const status = opts.status ?? 'succeeded';
  const r = s.startRun(taskId, 'executor');
  if (status !== 'started') s.finishRun(r.id, status);
  const started = new Date(BASE + (opts.startMs ?? 0)).toISOString();
  const ended = opts.durMs == null ? null : new Date(BASE + (opts.startMs ?? 0) + opts.durMs).toISOString();
  s.db.prepare(`UPDATE runs SET started_at = @s, ended_at = @e WHERE id = @id`)
    .run({ s: started, e: ended, id: r.id });
  return r.id;
}

// ──────────────── 纯函数 ────────────────
test('metrics 纯函数：percentile / ratio / estimateCostUnits', () => {
  assert.equal(percentile([], 50), 0);
  assert.equal(percentile([10], 95), 10);
  assert.equal(percentile([10, 20, 30, 40], 50), 20);   // ceil(0.5*4)=2 → idx 1
  assert.equal(percentile([10, 20, 30, 40], 95), 40);   // ceil(0.95*4)=4 → idx 3

  assert.equal(ratio(1, 2), 0.5);
  assert.equal(ratio(2, 3), 0.6667);
  assert.equal(ratio(0, 0), null);                       // 无样本不除零

  assert.equal(costPerMin('claude-opus-4-8'), 1.0);
  assert.equal(costPerMin('未知模型'), DEFAULT_COST_PER_MIN);
  assert.equal(estimateCostUnits(60_000, 'claude-opus-4-8'), 1.0);   // 1 分钟 × 1.0
  assert.equal(estimateCostUnits(120_000, 'claude-sonnet-4-6'), 0.4); // 2 分钟 × 0.2
});

// ──────────────── 空数据不崩 ────────────────
test('metrics 空数据：全 0 / null，不崩', () => {
  const s = fresh();
  const m = s.metrics();
  assert.equal(m.projectId, null);
  assert.equal(m.taskCount, 0);
  assert.deepEqual(m.duration, { count: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 });
  assert.equal(m.runs.successRate, null);
  assert.equal(m.review.reviewRate, null);
  assert.equal(m.review.securityRate, null);
  assert.equal(m.retry.retryRate, null);
  assert.equal(m.retry.needsAttention, 0);
  assert.deepEqual(m.byModel, []);
  assert.equal(m.estimated, true);
  s.close();
});

test('metrics：不存在的项目 → emptyMetrics（带 projectId）', () => {
  const s = fresh();
  const m = s.metrics('prj_does_not_exist');
  assert.deepEqual(m, emptyMetrics('prj_does_not_exist'));
  s.close();
});

// ──────────────── 执行时长 ────────────────
test('metrics 执行时长：avg/p50/p95；未结束与负时长不计入', () => {
  const s = fresh();
  const p = s.createProject({ name: 'd', repoPath: '/tmp/d-' + Math.random() });
  const t = s.createTask({ projectId: p.id, title: 'x', complexity: 'easy' });
  mkRun(s, t.id, { status: 'succeeded', startMs: 0, durMs: 1000 });
  mkRun(s, t.id, { status: 'succeeded', startMs: 10_000, durMs: 2000 });
  mkRun(s, t.id, { status: 'succeeded', startMs: 20_000, durMs: 3000 });
  mkRun(s, t.id, { status: 'started', startMs: 30_000, durMs: null });   // 未结束 → 不计
  mkRun(s, t.id, { status: 'failed', startMs: 40_000, durMs: -500 });    // 负时长 → 不计

  const m = s.metrics(p.id);
  assert.equal(m.duration.count, 3);
  assert.equal(m.duration.avgMs, 2000);
  assert.equal(m.duration.p50Ms, 2000);
  assert.equal(m.duration.p95Ms, 3000);
  assert.equal(m.duration.maxMs, 3000);
  s.close();
});

// ──────────────── 成功率 ────────────────
test('metrics 成功率：仅 succeeded/failed 计入，started/cancelled 不计', () => {
  const s = fresh();
  const p = s.createProject({ name: 's', repoPath: '/tmp/s-' + Math.random() });
  const t = s.createTask({ projectId: p.id, title: 'x', complexity: 'easy' });
  mkRun(s, t.id, { status: 'succeeded', durMs: 1000 });
  mkRun(s, t.id, { status: 'succeeded', durMs: 1000 });
  mkRun(s, t.id, { status: 'failed', durMs: 1000 });
  mkRun(s, t.id, { status: 'cancelled', durMs: 1000 });
  mkRun(s, t.id, { status: 'started', durMs: null });

  const m = s.metrics(p.id);
  assert.equal(m.runs.total, 3);
  assert.equal(m.runs.succeeded, 2);
  assert.equal(m.runs.failed, 1);
  assert.equal(m.runs.successRate, 0.6667);
  s.close();
});

// ──────────────── 复审通过率 ────────────────
test('metrics 复审通过率：verdict / securityVerdict = approve 占比', () => {
  const s = fresh();
  const p = s.createProject({ name: 'r', repoPath: '/tmp/r-' + Math.random() });
  const base = { branch: null, worktree: null, diffSummary: null, commits: [], prUrl: null, summary: null, securitySummary: null, mergeTaskId: null };
  const t1 = s.createTask({ projectId: p.id, title: 'a', complexity: 'easy' });
  const t2 = s.createTask({ projectId: p.id, title: 'b', complexity: 'easy' });
  const t3 = s.createTask({ projectId: p.id, title: 'c', complexity: 'easy' });
  s.setResult(t1.id, { ...base, verdict: 'approve', securityVerdict: 'approve' });
  s.setResult(t2.id, { ...base, verdict: 'approve', securityVerdict: 'reject' });
  s.setResult(t3.id, { ...base, verdict: 'reject', securityVerdict: null });   // 安全未审 → 不计

  const m = s.metrics(p.id);
  assert.equal(m.review.reviewTotal, 3);
  assert.equal(m.review.reviewApprove, 2);
  assert.equal(m.review.reviewRate, 0.6667);
  assert.equal(m.review.securityTotal, 2);
  assert.equal(m.review.securityApprove, 1);
  assert.equal(m.review.securityRate, 0.5);
  s.close();
});

// ──────────────── 重试率 / needs_attention ────────────────
test('metrics 重试率：failed→queued 次数 / 任务数；needs_attention 计数', () => {
  const s = fresh();
  const p = s.createProject({ name: 'rt', repoPath: '/tmp/rt-' + Math.random() });
  // 任务 A：跑两轮失败重试（failed→queued ×2）
  const a = s.createTask({ projectId: p.id, title: 'a', complexity: 'easy' }); // ready
  s.transition(a.id, 'queued');
  s.transition(a.id, 'executing');
  s.transition(a.id, 'failed');
  s.transition(a.id, 'queued');        // 重试 1
  s.transition(a.id, 'executing');
  s.transition(a.id, 'failed');
  s.transition(a.id, 'queued');        // 重试 2
  // 任务 B：失败后升级 needs_attention（不是 failed→queued，不计重试）
  const b = s.createTask({ projectId: p.id, title: 'b', complexity: 'easy' });
  s.transition(b.id, 'queued');
  s.transition(b.id, 'executing');
  s.transition(b.id, 'failed');
  s.transition(b.id, 'needs_attention');

  const m = s.metrics(p.id);
  assert.equal(m.retry.retries, 2);
  assert.equal(m.retry.taskCount, 2);
  assert.equal(m.retry.retryRate, 1);          // 2 / 2
  assert.equal(m.retry.needsAttention, 1);
  s.close();
});

// ──────────────── 按模型额度估算 ────────────────
test('metrics 按模型额度：时长×档位 粗估，estimated=true', () => {
  const s = fresh();
  const p = s.createProject({ name: 'mdl', repoPath: '/tmp/mdl-' + Math.random(), model: 'claude-opus-4-8' });
  const t = s.createTask({ projectId: p.id, title: 'x', complexity: 'easy' });
  mkRun(s, t.id, { status: 'succeeded', startMs: 0, durMs: 60_000 });
  mkRun(s, t.id, { status: 'succeeded', startMs: 100_000, durMs: 60_000 });

  const m = s.metrics(p.id);
  assert.equal(m.byModel.length, 1);
  assert.deepEqual(m.byModel[0], {
    model: 'claude-opus-4-8', runs: 2, durationMs: 120_000, estCostUnits: 2.0, estimated: true,
  });
  assert.equal(m.estimated, true);
  s.close();
});

// ──────────────── 项目过滤 ────────────────
test('metrics 项目过滤：只统计指定项目', () => {
  const s = fresh();
  const pa = s.createProject({ name: 'A', repoPath: '/tmp/A-' + Math.random() });
  const pb = s.createProject({ name: 'B', repoPath: '/tmp/B-' + Math.random() });
  const ta = s.createTask({ projectId: pa.id, title: 'a', complexity: 'easy' });
  const tb = s.createTask({ projectId: pb.id, title: 'b', complexity: 'easy' });
  mkRun(s, ta.id, { status: 'succeeded', durMs: 1000 });
  mkRun(s, tb.id, { status: 'failed', durMs: 1000 });

  const ma = s.metrics(pa.id);
  assert.equal(ma.taskCount, 1);
  assert.equal(ma.runs.succeeded, 1);
  assert.equal(ma.runs.failed, 0);

  const all = s.metrics();
  assert.equal(all.taskCount, 2);
  assert.equal(all.runs.total, 2);
  s.close();
});