feat(eval): ASR 模型评估框架
横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer)， 重点覆盖中英混说，产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,200 @@
|
||||
"""命令行入口：python -m asr_eval run --config config.yaml
|
||||
|
||||
流程：载 config → 物料化样本 → 逐引擎逐样本识别(本地引擎包资源探针) → 算分 → 出报告。
|
||||
gummy 的 API key：优先 env DASHSCOPE_API_KEY，否则读 stdin 首行
|
||||
（遵全局规则：rbw get dashscope-api-key | python -m asr_eval run ...）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import yaml
|
||||
from rich.console import Console
|
||||
from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn
|
||||
|
||||
from .datasets import build_dataset
|
||||
from .engines import build_engine
|
||||
from .metrics.err import score_sample
|
||||
from .metrics.resource import ResourceProbe
|
||||
from .report import aggregate, render_markdown, write_reports
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def _resolve_api_key() -> str:
|
||||
key = os.environ.get("DASHSCOPE_API_KEY", "").strip()
|
||||
if key:
|
||||
return key
|
||||
if not sys.stdin.isatty():
|
||||
line = sys.stdin.readline().strip()
|
||||
if line:
|
||||
return line
|
||||
return ""
|
||||
|
||||
|
||||
def _load_samples(cfg: dict, only: set[str] | None, limit: int | None) -> list:
|
||||
audio_cache = cfg.get("audio_cache", "datasets/_audio_cache")
|
||||
samples = []
|
||||
for dcfg in cfg.get("datasets", []):
|
||||
if only and dcfg["name"] not in only:
|
||||
continue
|
||||
ds = build_dataset(dcfg, audio_cache)
|
||||
console.print(f"[cyan]载入数据集[/] {dcfg['name']} (type={dcfg['type']}) ...")
|
||||
try:
|
||||
got = list(ds.samples(limit=limit))
|
||||
except Exception as ex:
|
||||
console.print(f"[red]数据集 {dcfg['name']} 载入失败，跳过：{ex}[/]")
|
||||
continue
|
||||
console.print(f" → {len(got)} 条")
|
||||
samples.extend(got)
|
||||
return samples
|
||||
|
||||
|
||||
def _enabled_engines(cfg: dict, only: set[str] | None, api_key: str) -> list[dict]:
|
||||
out = []
|
||||
for ecfg in cfg.get("engines", []):
|
||||
if not ecfg.get("enabled", True):
|
||||
continue
|
||||
if only and ecfg["name"] not in only:
|
||||
continue
|
||||
if ecfg["type"] == "gummy":
|
||||
ecfg = {**ecfg, "api_key": api_key}
|
||||
out.append(ecfg)
|
||||
return out
|
||||
|
||||
|
||||
def cmd_run(args: argparse.Namespace) -> int:
|
||||
with open(args.config, encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
|
||||
limit = args.sample_limit if args.sample_limit is not None else cfg.get("sample_limit")
|
||||
only_engines = set(args.engines.split(",")) if args.engines else None
|
||||
only_datasets = set(args.datasets.split(",")) if args.datasets else None
|
||||
|
||||
engine_cfgs = _enabled_engines(cfg, only_engines, "")
|
||||
need_key = any(e["type"] == "gummy" for e in engine_cfgs)
|
||||
api_key = _resolve_api_key() if need_key else ""
|
||||
if need_key and not api_key:
|
||||
console.print("[red]gummy 已启用但未取到 API key（设 DASHSCOPE_API_KEY 或 stdin 传入）[/]")
|
||||
return 2
|
||||
engine_cfgs = _enabled_engines(cfg, only_engines, api_key)
|
||||
|
||||
samples = _load_samples(cfg, only_datasets, limit)
|
||||
if not samples:
|
||||
console.print("[red]没有可评估的样本，检查 datasets 配置或 manifest[/]")
|
||||
return 1
|
||||
|
||||
rows: list[dict] = []
|
||||
engines_meta: dict[str, dict] = {}
|
||||
|
||||
for ecfg in engine_cfgs:
|
||||
name = ecfg["name"]
|
||||
console.print(f"\n[bold magenta]引擎 {name}[/] (type={ecfg['type']}) 加载中 ...")
|
||||
engine = build_engine(ecfg)
|
||||
try:
|
||||
engine.load()
|
||||
except Exception as ex:
|
||||
console.print(f"[red]引擎 {name} 加载失败，跳过：{ex}[/]")
|
||||
continue
|
||||
|
||||
engines_meta[name] = {
|
||||
"is_local": engine.is_local,
|
||||
"kind": engine.kind,
|
||||
"model_size_mb": engine.model_size_mb(),
|
||||
"cost_per_min": engine.cost_per_min(),
|
||||
}
|
||||
|
||||
with Progress(
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("{task.completed}/{task.total}"),
|
||||
TimeElapsedColumn(),
|
||||
console=console,
|
||||
) as prog:
|
||||
task = prog.add_task(f"{name}", total=len(samples))
|
||||
for s in samples:
|
||||
tr = _transcribe_one(engine, s.audio_path)
|
||||
sc = score_sample(s.ref_text, tr.text)
|
||||
rows.append({
|
||||
"engine": name,
|
||||
"dataset": s.dataset,
|
||||
"category": s.category,
|
||||
"lang": s.lang,
|
||||
"domain": s.domain,
|
||||
"sample_id": s.id,
|
||||
"ref": s.ref_text,
|
||||
"hyp": tr.text,
|
||||
"counts": sc,
|
||||
"audio_sec": tr.audio_sec,
|
||||
"proc_sec": tr.proc_sec,
|
||||
"rtf": tr.rtf,
|
||||
"first_partial_sec": tr.first_partial_sec,
|
||||
"finalize_sec": tr.finalize_sec,
|
||||
"peak_rss_mb": tr.peak_rss_mb,
|
||||
"avg_cpu": tr.avg_cpu,
|
||||
"error": tr.error,
|
||||
})
|
||||
prog.advance(task)
|
||||
engine.unload()
|
||||
|
||||
if not rows:
|
||||
console.print("[red]无结果（所有引擎都失败了？）[/]")
|
||||
return 1
|
||||
|
||||
report = aggregate(rows, engines_meta)
|
||||
out_dir = os.path.join(cfg.get("output_dir", "results"), datetime.now().strftime("%Y%m%d-%H%M%S"))
|
||||
paths = write_reports(report, out_dir)
|
||||
_dump_raw(rows, out_dir)
|
||||
|
||||
console.print("\n" + render_markdown(report))
|
||||
console.print(f"\n[green]报告已写入[/] {out_dir}/ (report.json / report.md / report.html)")
|
||||
return 0
|
||||
|
||||
|
||||
def _transcribe_one(engine, audio_path: str):
|
||||
"""本地引擎包资源探针；云端引擎直接调用。"""
|
||||
if engine.is_local:
|
||||
with ResourceProbe() as probe:
|
||||
tr = engine.transcribe(audio_path)
|
||||
if tr.peak_rss_mb is None:
|
||||
tr.peak_rss_mb = probe.peak_rss_mb
|
||||
if tr.avg_cpu is None:
|
||||
tr.avg_cpu = probe.avg_cpu
|
||||
return tr
|
||||
return engine.transcribe(audio_path)
|
||||
|
||||
|
||||
def _dump_raw(rows: list[dict], out_dir: str) -> None:
|
||||
import json
|
||||
|
||||
path = os.path.join(out_dir, "results.jsonl")
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
for r in rows:
|
||||
rec = {k: v for k, v in r.items() if k != "counts"}
|
||||
rec["cer"] = r["counts"]["cer"].rate
|
||||
rec["wer"] = r["counts"]["wer"].rate
|
||||
rec["mer"] = r["counts"]["mer"].rate
|
||||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(prog="asr_eval", description="dudu ASR 模型评估框架")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
run = sub.add_parser("run", help="跑评估")
|
||||
run.add_argument("--config", default="config.yaml")
|
||||
run.add_argument("--engines", help="只跑这些引擎（逗号分隔，覆盖 enabled）")
|
||||
run.add_argument("--datasets", help="只跑这些数据集（逗号分隔）")
|
||||
run.add_argument("--sample-limit", type=int, default=None, help="每个数据集采样上限（覆盖 config）")
|
||||
run.set_defaults(func=cmd_run)
|
||||
|
||||
args = ap.parse_args()
|
||||
sys.exit(args.func(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user