maestro/test/models.test.ts

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { pickModel, resolveModel, isModelError, pickFallbackModel, MODEL_FALLBACK_CHAIN, DEFAULT_MODEL } from '../src/executor/models.js';
import type { Project, Task, ProjectModels } from '../src/model/types.js';
import type { Complexity } from '../src/model/complexity.js';

function fakeTask(complexity: Complexity): Task {
  return { complexity } as Task;
}
function fakeProject(over: Partial<Project> = {}): Project {
  return { model: null, models: null, ...over } as Project;
}

/** 临时设置 env，跑完恢复（避免污染其他用例） */
function withEnv(vars: Record<string, string>, fn: () => void): void {
  const saved = new Map<string, string | undefined>();
  for (const [k, v] of Object.entries(vars)) {
    saved.set(k, process.env[k]);
    process.env[k] = v;
  }
  try {
    fn();
  } finally {
    for (const [k, v] of saved) {
      if (v === undefined) delete process.env[k];
      else process.env[k] = v;
    }
  }
}

const ENV_KEYS = [
  'MAESTRO_MODEL_EASY', 'MAESTRO_MODEL_MEDIUM', 'MAESTRO_MODEL_HARD',
  'MAESTRO_MODEL_REVIEW_EASY', 'MAESTRO_MODEL_REVIEW_MEDIUM', 'MAESTRO_MODEL_REVIEW_HARD',
  'MAESTRO_MODEL_PLAN_EASY', 'MAESTRO_MODEL_PLAN_MEDIUM', 'MAESTRO_MODEL_PLAN_HARD',
];
const clearEnv = (): Record<string, string> => Object.fromEntries(ENV_KEYS.map((k) => [k, '']));

test('默认档：所有角色 × 所有复杂度统一 opus-4.8（无 project/env 配置）', () => {
  withEnv(clearEnv(), () => {
    const p = fakeProject();
    assert.equal(DEFAULT_MODEL, 'claude-opus-4-8');
    for (const role of ['executor', 'planner', 'reviewer', 'conflict'] as const) {
      for (const c of ['easy', 'medium', 'hard'] as Complexity[]) {
        assert.equal(pickModel(fakeTask(c), p, role), 'claude-opus-4-8', `${role}/${c}`);
      }
    }
  });
});

test('project.models：按角色覆盖默认（字符串=全复杂度统一）', () => {
  withEnv(clearEnv(), () => {
    const models: ProjectModels = { reviewer: 'claude-fable-5', conflict: 'claude-fable-5' };
    const p = fakeProject({ models });
    // reviewer/conflict 被项目级 models 覆盖成 fable-5（含三档）
    for (const c of ['easy', 'medium', 'hard'] as Complexity[]) {
      assert.equal(pickModel(fakeTask(c), p, 'reviewer'), 'claude-fable-5');
      assert.equal(pickModel(fakeTask(c), p, 'conflict'), 'claude-fable-5');
    }
    // executor 未配 → 仍默认 opus
    assert.equal(pickModel(fakeTask('easy'), p, 'executor'), 'claude-opus-4-8');
  });
});

test('project.models：按复杂度分档（对象形式），缺档回落默认', () => {
  withEnv(clearEnv(), () => {
    const models: ProjectModels = { executor: { easy: 'claude-sonnet-4-6', hard: 'claude-fable-5' } };
    const p = fakeProject({ models });
    assert.equal(pickModel(fakeTask('easy'), p, 'executor'), 'claude-sonnet-4-6');
    assert.equal(pickModel(fakeTask('hard'), p, 'executor'), 'claude-fable-5');
    // medium 未在对象里 → 回落默认 opus
    assert.equal(pickModel(fakeTask('medium'), p, 'executor'), 'claude-opus-4-8');
  });
});

test('优先级：project.models > 旧 project.model > env > 默认', () => {
  withEnv({ ...clearEnv(), MAESTRO_MODEL_MEDIUM: 'claude-haiku-4-5' }, () => {
    // models.executor 覆盖一切
    const p1 = fakeProject({ model: 'claude-opus-4-6', models: { executor: 'claude-sonnet-4-6' } });
    assert.equal(pickModel(fakeTask('medium'), p1, 'executor'), 'claude-sonnet-4-6');
    // 无 models → 旧 project.model 覆盖 env
    const p2 = fakeProject({ model: 'claude-opus-4-6' });
    assert.equal(pickModel(fakeTask('medium'), p2, 'executor'), 'claude-opus-4-6');
    // 无 models / 无 project.model → env 覆盖默认
    const p3 = fakeProject();
    assert.equal(pickModel(fakeTask('medium'), p3, 'executor'), 'claude-haiku-4-5');
  });
});

test('旧 project.model 不覆盖 reviewer/conflict（保自审独立性），但 project.models 可以', () => {
  withEnv(clearEnv(), () => {
    const p = fakeProject({ model: 'claude-opus-4-6' });
    // project.model 只作用 executor/planner
    assert.equal(pickModel(fakeTask('hard'), p, 'executor'), 'claude-opus-4-6');
    assert.equal(pickModel(fakeTask('hard'), p, 'planner'), 'claude-opus-4-6');
    // reviewer/conflict 不受 project.model 影响 → 仍默认 opus
    assert.equal(pickModel(fakeTask('hard'), p, 'reviewer'), 'claude-opus-4-8');
    assert.equal(pickModel(fakeTask('hard'), p, 'conflict'), 'claude-opus-4-8');
    // 但显式 project.models.reviewer 可以覆盖
    const p2 = fakeProject({ model: 'claude-opus-4-6', models: { reviewer: 'claude-sonnet-4-6' } });
    assert.equal(pickModel(fakeTask('hard'), p2, 'reviewer'), 'claude-sonnet-4-6');
  });
});

test('resolveModel：env 空白字符串视为未设置', () => {
  withEnv({ ...clearEnv(), MAESTRO_MODEL_MEDIUM: '   ' }, () => {
    assert.equal(resolveModel(fakeProject(), 'executor', 'medium'), 'claude-opus-4-8');
  });
});

test('isModelError：模型不可用类错误才触发回退', () => {
  // 应触发
  assert.ok(isModelError('model claude-fable-5 not_found'));
  assert.ok(isModelError('Invalid model: claude-x'));
  assert.ok(isModelError('permission denied for model claude-fable-5'));
  assert.ok(isModelError('404 model does not exist'));
  assert.ok(isModelError('The requested model is unavailable'));
  assert.ok(isModelError('unknown model "claude-fable-5"'));
  // Claude Code 的「未开通某档模型」报错（线上 fable-5 实测）
  assert.ok(isModelError("There's an issue with the selected model (claude-fable-5)"));
  // 不应触发
  assert.ok(!isModelError('执行超时'));
  assert.ok(!isModelError('network connection refused'));
  assert.ok(!isModelError('not found: /tmp/some/file'));            // 没提到 model
  assert.ok(!isModelError('model produced empty output'));          // 提到 model 但不是可用性错误
});

test('pickFallbackModel：回退链取下一档（opus→sonnet→fable→opus，opus 置链首=默认档优先）', () => {
  assert.deepEqual([...MODEL_FALLBACK_CHAIN], ['claude-opus-4-8', 'claude-sonnet-4-6', 'claude-fable-5']);
  assert.equal(pickFallbackModel('claude-opus-4-8'), 'claude-sonnet-4-6');
  assert.equal(pickFallbackModel('claude-sonnet-4-6'), 'claude-fable-5');
  assert.equal(pickFallbackModel('claude-fable-5'), 'claude-opus-4-8');
});