feat(eval): ASR 模型评估框架

横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer)，重点覆盖中英混说，产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-13 11:25:04 +08:00
parent 40760aa884
commit 25acf9db6e
25 changed files with 1594 additions and 0 deletions
@@ -0,0 +1,25 @@
+"""数据集注册表：config 的 type(hf|custom) 映射到适配器。
+
+公共集（ASCEND/AISHELL/LibriSpeech）统一走 HFDataset，差异全在 config.yaml 里描述
+（hf_id/split/text_field/lang），所以"广覆盖 + 自定义"只需改配置，不必加代码。
+"""
+
+from __future__ import annotations
+
+from .base import Dataset
+
+
+def build_dataset(cfg: dict, audio_cache: str) -> Dataset:
+    dtype = cfg["type"]
+    if dtype == "hf":
+        from .hf import HFDataset
+
+        return HFDataset(cfg, audio_cache)
+    if dtype == "custom":
+        from .custom import CustomDataset
+
+        return CustomDataset(cfg)
+    raise ValueError(f"未知数据集类型: {dtype}")
+
+
+__all__ = ["Dataset", "build_dataset"]
@@ -0,0 +1,34 @@
+"""数据集适配器基类与音频物料化工具。"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+
+import numpy as np
+import soundfile as sf
+
+from ..manifest import Sample
+
+TARGET_SR = 16000
+
+
+class Dataset:
+    name: str = "base"
+
+    def samples(self, limit: int | None = None) -> Iterator[Sample]:
+        raise NotImplementedError
+
+
+def write_wav_16k(array: np.ndarray, sr: int, out_path: str) -> None:
+    """把（可能任意采样率的）单/多声道数组写成 16k/mono/16bit wav。"""
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    data = np.asarray(array, dtype=np.float32)
+    if data.ndim > 1:
+        data = data.mean(axis=1)
+    if sr != TARGET_SR:
+        import soxr
+
+        data = soxr.resample(data, sr, TARGET_SR)
+    data = np.clip(data, -1.0, 1.0)
+    sf.write(out_path, (data * 32767.0).astype(np.int16), TARGET_SR, subtype="PCM_16")
@@ -0,0 +1,62 @@
+"""自定义集：JSONL manifest，每行一条样本。
+
+格式（audio 路径相对 manifest 文件所在目录解析）：
+  {"audio": "clips/a.wav", "text": "帮我把这份 weekly report 整理一下", "lang": "zh-en", "domain": "tech"}
+
+lang 缺省按文本自动猜（含中文且含 ASCII 字母→zh-en；纯 ASCII→en；否则 zh）。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Iterator
+
+from ..manifest import Sample
+
+_CJK = re.compile(r"[一-鿿]")
+_ASCII_ALPHA = re.compile(r"[A-Za-z]")
+
+
+def _guess_lang(text: str) -> str:
+    has_cjk = bool(_CJK.search(text))
+    has_en = bool(_ASCII_ALPHA.search(text))
+    if has_cjk and has_en:
+        return "zh-en"
+    if has_en and not has_cjk:
+        return "en"
+    return "zh"
+
+
+class CustomDataset:
+    def __init__(self, cfg: dict):
+        self.name = cfg.get("name", "custom")
+        self.manifest = cfg["manifest"]
+        self.domain_default = cfg.get("domain", "general")
+        self._base = os.path.dirname(os.path.abspath(self.manifest))
+
+    def samples(self, limit: int | None = None) -> Iterator[Sample]:
+        with open(self.manifest, encoding="utf-8") as f:
+            n = 0
+            for i, line in enumerate(f):
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                if limit is not None and n >= limit:
+                    break
+                row = json.loads(line)
+                text = (row.get("text") or "").strip()
+                audio = row["audio"]
+                if not os.path.isabs(audio):
+                    audio = os.path.join(self._base, audio)
+                lang = row.get("lang") or _guess_lang(text)
+                n += 1
+                yield Sample(
+                    id=row.get("id", f"{self.name}-{i:06d}"),
+                    audio_path=audio,
+                    ref_text=text,
+                    lang=lang,
+                    domain=row.get("domain", self.domain_default),
+                    dataset=self.name,
+                )
@@ -0,0 +1,89 @@
+"""通用 HuggingFace 数据集适配器（流式）。
+
+config 条目示例：
+  - name: ascend
+    type: hf
+    hf_id: CAiRE/ASCEND
+    split: test
+    text_field: transcription
+    audio_field: audio          # 默认 audio
+    lang: zh-en
+    domain: general
+    trust_remote_code: false
+
+流式（streaming=True）避免为跑小样而下载整个数据集；每条把音频物料化为 16k wav 缓存。
+"""
+
+from __future__ import annotations
+
+import io
+import os
+from collections.abc import Iterator
+
+import soundfile as sf
+
+from ..manifest import Sample
+from .base import Dataset, write_wav_16k
+
+
+class HFDataset(Dataset):
+    def __init__(self, cfg: dict, audio_cache: str):
+        self.name = cfg["name"]
+        self.hf_id = cfg["hf_id"]
+        self.hf_config = cfg.get("hf_config")
+        self.split = cfg.get("split", "test")
+        self.text_field = cfg["text_field"]
+        self.audio_field = cfg.get("audio_field", "audio")
+        self.lang = cfg.get("lang", "zh")
+        self.domain = cfg.get("domain", "general")
+        self.trust_remote_code = cfg.get("trust_remote_code", False)
+        self.cache_dir = os.path.join(audio_cache, self.name)
+
+    def samples(self, limit: int | None = None) -> Iterator[Sample]:
+        from datasets import Audio, load_dataset
+
+        ds = load_dataset(
+            self.hf_id,
+            self.hf_config,
+            split=self.split,
+            streaming=True,
+            trust_remote_code=self.trust_remote_code,
+        )
+        # 关闭自动解码（新版 datasets 需 torchcodec）：拿原始 bytes/path 自己用 soundfile 读，避开重依赖。
+        ds = ds.cast_column(self.audio_field, Audio(decode=False))
+        n = 0
+        for i, row in enumerate(ds):
+            if limit is not None and n >= limit:
+                break
+            text = (row.get(self.text_field) or "").strip()
+            if not text:
+                continue
+            audio = row.get(self.audio_field)
+            if not audio:
+                continue
+            wav_path = os.path.join(self.cache_dir, f"{i:06d}.wav")
+            if not os.path.exists(wav_path):
+                array, sr = self._decode(audio)
+                if array is None:
+                    continue
+                write_wav_16k(array, sr, wav_path)
+            n += 1
+            yield Sample(
+                id=f"{self.name}-{i:06d}",
+                audio_path=wav_path,
+                ref_text=text,
+                lang=self.lang,
+                domain=self.domain,
+                dataset=self.name,
+            )
+
+    @staticmethod
+    def _decode(audio: dict):
+        """从未解码的 HF 音频列读出 (array, sr)：优先 bytes，其次本地 path。"""
+        if audio.get("bytes"):
+            data, sr = sf.read(io.BytesIO(audio["bytes"]), dtype="float32", always_2d=False)
+            return data, sr
+        if audio.get("path") and os.path.exists(audio["path"]):
+            data, sr = sf.read(audio["path"], dtype="float32", always_2d=False)
+            return data, sr
+        return None, None