maestro/test/models.test.ts

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { pickModel, isModelError, pickFallbackModel, MODEL_FALLBACK_CHAIN } from '../src/executor/models.js';
import type { Project, Task } from '../src/model/types.js';
import type { Complexity } from '../src/model/complexity.js';

function fakeTask(complexity: Complexity): Task {
  return { complexity } as Task;
}
function fakeProject(model: string | null = null): Project {
  return { model } as Project;
}

/** 临时设置 env，跑完恢复（避免污染其他用例） */
function withEnv(vars: Record<string, string>, fn: () => void): void {
  const saved = new Map<string, string | undefined>();
  for (const [k, v] of Object.entries(vars)) {
    saved.set(k, process.env[k]);
    process.env[k] = v;
  }
  try {
    fn();
  } finally {
    for (const [k, v] of saved) {
      if (v === undefined) delete process.env[k];
      else process.env[k] = v;
    }
  }
}

const ENV_KEYS = [
  'MAESTRO_MODEL_EASY', 'MAESTRO_MODEL_MEDIUM', 'MAESTRO_MODEL_HARD',
  'MAESTRO_MODEL_REVIEW_EASY', 'MAESTRO_MODEL_REVIEW_MEDIUM', 'MAESTRO_MODEL_REVIEW_HARD',
];

test('pickModel：复杂度映射默认值（executor / reviewer）', () => {
  // 防止外部 env 干扰：先清空再断言默认
  withEnv(Object.fromEntries(ENV_KEYS.map((k) => [k, ''])), () => {
    const p = fakeProject();
    assert.equal(pickModel(fakeTask('easy'), p, 'executor'), 'claude-sonnet-4-6');
    assert.equal(pickModel(fakeTask('medium'), p, 'executor'), 'claude-opus-4-8');
    assert.equal(pickModel(fakeTask('hard'), p, 'executor'), 'claude-opus-4-8');
    // reviewer 三档统一 fable-5（复审是质量关口，用最强档保严格度）
    assert.equal(pickModel(fakeTask('easy'), p, 'reviewer'), 'claude-fable-5');
    assert.equal(pickModel(fakeTask('medium'), p, 'reviewer'), 'claude-fable-5');
    assert.equal(pickModel(fakeTask('hard'), p, 'reviewer'), 'claude-fable-5');
    // planner hard 拆解最烧脑 → fable-5；medium=opus；easy=sonnet
    assert.equal(pickModel(fakeTask('hard'), p, 'planner'), 'claude-fable-5');
    assert.equal(pickModel(fakeTask('medium'), p, 'planner'), 'claude-opus-4-8');
    assert.equal(pickModel(fakeTask('easy'), p, 'planner'), 'claude-sonnet-4-6');
  });
});

test('pickModel：project.model executor 最优先，但 reviewer/conflict 不受 project.model 覆盖', () => {
  withEnv(Object.fromEntries(ENV_KEYS.map((k) => [k, ''])), () => {
    const p = fakeProject('claude-opus-4-6');
    for (const c of ['easy', 'medium', 'hard'] as const) {
      // executor 受 project.model 覆盖
      assert.equal(pickModel(fakeTask(c), p, 'executor'), 'claude-opus-4-6');
      // reviewer 不受 project.model 覆盖，始终用 fable-5（保自审独立性）
      assert.equal(pickModel(fakeTask(c), p, 'reviewer'), 'claude-fable-5');
      // conflict 固定 fable-5（解冲突恒用最强档）
      assert.equal(pickModel(fakeTask(c), p, 'conflict'), 'claude-fable-5');
    }
  });
});

test('pickModel：env 覆盖默认值，executor/reviewer 各用各的 env', () => {
  withEnv({
    MAESTRO_MODEL_EASY: 'claude-haiku-4-5',
    MAESTRO_MODEL_REVIEW_EASY: 'claude-sonnet-4-6',  // reviewer env 可单独覆盖
  }, () => {
    const p = fakeProject();
    assert.equal(pickModel(fakeTask('easy'), p, 'executor'), 'claude-haiku-4-5');
    // reviewer easy 被 MAESTRO_MODEL_REVIEW_EASY 覆盖
    assert.equal(pickModel(fakeTask('easy'), p, 'reviewer'), 'claude-sonnet-4-6');
    // reviewer hard 无 env → 默认 fable-5
    assert.equal(pickModel(fakeTask('hard'), p, 'reviewer'), 'claude-fable-5');
    assert.equal(pickModel(fakeTask('hard'), p, 'executor'), 'claude-opus-4-8');
  });
});

test('pickModel：env 为空白字符串视为未设置', () => {
  withEnv({ MAESTRO_MODEL_MEDIUM: '   ' }, () => {
    assert.equal(pickModel(fakeTask('medium'), fakeProject(), 'executor'), 'claude-opus-4-8');
  });
});

test('isModelError：模型不可用类错误才触发回退', () => {
  // 应触发
  assert.ok(isModelError('model claude-fable-5 not_found'));
  assert.ok(isModelError('Invalid model: claude-x'));
  assert.ok(isModelError('permission denied for model claude-fable-5'));
  assert.ok(isModelError('404 model does not exist'));
  assert.ok(isModelError('The requested model is unavailable'));
  assert.ok(isModelError('unknown model "claude-fable-5"'));
  // 不应触发
  assert.ok(!isModelError('执行超时'));
  assert.ok(!isModelError('network connection refused'));
  assert.ok(!isModelError('not found: /tmp/some/file'));            // 没提到 model
  assert.ok(!isModelError('model produced empty output'));          // 提到 model 但不是可用性错误
});

test('pickFallbackModel：回退链取下一档（循环）', () => {
  // 链：fable → opus → sonnet → fable（循环，fable 置链首=优先最强档）
  assert.deepEqual([...MODEL_FALLBACK_CHAIN], ['claude-fable-5', 'claude-opus-4-8', 'claude-sonnet-4-6']);
  assert.equal(pickFallbackModel('claude-fable-5'), 'claude-opus-4-8');
  assert.equal(pickFallbackModel('claude-opus-4-8'), 'claude-sonnet-4-6');
  assert.equal(pickFallbackModel('claude-sonnet-4-6'), 'claude-fable-5');
});