feat(eval): ASR 模型评估框架

横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer)，重点覆盖中英混说，产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-13 11:25:04 +08:00
parent 40760aa884
commit 25acf9db6e
25 changed files with 1594 additions and 0 deletions
@@ -0,0 +1,62 @@
+"""自定义集：JSONL manifest，每行一条样本。
+
+格式（audio 路径相对 manifest 文件所在目录解析）：
+  {"audio": "clips/a.wav", "text": "帮我把这份 weekly report 整理一下", "lang": "zh-en", "domain": "tech"}
+
+lang 缺省按文本自动猜（含中文且含 ASCII 字母→zh-en；纯 ASCII→en；否则 zh）。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Iterator
+
+from ..manifest import Sample
+
+_CJK = re.compile(r"[一-鿿]")
+_ASCII_ALPHA = re.compile(r"[A-Za-z]")
+
+
+def _guess_lang(text: str) -> str:
+    has_cjk = bool(_CJK.search(text))
+    has_en = bool(_ASCII_ALPHA.search(text))
+    if has_cjk and has_en:
+        return "zh-en"
+    if has_en and not has_cjk:
+        return "en"
+    return "zh"
+
+
+class CustomDataset:
+    def __init__(self, cfg: dict):
+        self.name = cfg.get("name", "custom")
+        self.manifest = cfg["manifest"]
+        self.domain_default = cfg.get("domain", "general")
+        self._base = os.path.dirname(os.path.abspath(self.manifest))
+
+    def samples(self, limit: int | None = None) -> Iterator[Sample]:
+        with open(self.manifest, encoding="utf-8") as f:
+            n = 0
+            for i, line in enumerate(f):
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                if limit is not None and n >= limit:
+                    break
+                row = json.loads(line)
+                text = (row.get("text") or "").strip()
+                audio = row["audio"]
+                if not os.path.isabs(audio):
+                    audio = os.path.join(self._base, audio)
+                lang = row.get("lang") or _guess_lang(text)
+                n += 1
+                yield Sample(
+                    id=row.get("id", f"{self.name}-{i:06d}"),
+                    audio_path=audio,
+                    ref_text=text,
+                    lang=lang,
+                    domain=row.get("domain", self.domain_default),
+                    dataset=self.name,
+                )