feat(eval): ASR 模型评估框架

横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer),
重点覆盖中英混说,产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小)
对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。
gummy 引擎对照 server/internal/asr/gummy.go 协议移植。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-13 11:25:04 +08:00
parent 40760aa884
commit 25acf9db6e
25 changed files with 1594 additions and 0 deletions
+70
View File
@@ -0,0 +1,70 @@
"""本地 faster-whisper 引擎(CTranslate2CPU-first)。多语,对中英混说有基础能力。
依赖:pip install '.[whisper]'
"""
from __future__ import annotations
import time
from ..metrics.resource import dir_size_mb
from .base import Engine, Transcript
class WhisperEngine(Engine):
kind = "offline"
is_local = True
def __init__(
self,
name: str = "whisper-small",
model_size: str = "small",
device: str = "cpu",
compute_type: str = "int8",
model_dir: str | None = None,
):
self.name = name
self.model_size = model_size
self.device = device
self.compute_type = compute_type
self.model_dir = model_dir
self.model = None
self._weights_path: str | None = None
def load(self) -> None:
from faster_whisper import WhisperModel
self.model = WhisperModel(
self.model_size,
device=self.device,
compute_type=self.compute_type,
download_root=self.model_dir,
)
self._resolve_weights_path()
def _resolve_weights_path(self) -> None:
# 解析磁盘权重路径用于报模型大小(best-effort
try:
from huggingface_hub import snapshot_download
repo = f"Systran/faster-whisper-{self.model_size}"
self._weights_path = snapshot_download(repo, local_files_only=True, cache_dir=self.model_dir)
except Exception:
self._weights_path = self.model_dir
def transcribe(self, audio_path: str) -> Transcript:
if self.model is None:
self.load()
t0 = time.monotonic()
# language=None → 自动检测;中英混说交给模型自身
segments, info = self.model.transcribe(audio_path, language=None, beam_size=5)
text = "".join(seg.text for seg in segments) # 生成器在此 join 时才真正解码
proc = time.monotonic() - t0
return Transcript(text=text.strip(), audio_sec=float(info.duration), proc_sec=proc)
def model_size_mb(self) -> float | None:
size = dir_size_mb(self._weights_path) if self._weights_path else 0.0
return size or None
def unload(self) -> None:
self.model = None