feat(eval): ASR 模型评估框架
横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer)， 重点覆盖中英混说，产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,45 @@
|
||||
"""引擎注册表：config.yaml 里的 type 映射到具体引擎类（懒加载，避免未装的重依赖被导入）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .base import Engine, Transcript
|
||||
|
||||
|
||||
def build_engine(cfg: dict) -> Engine:
|
||||
"""按 config 的单个 engine 条目构造引擎实例。"""
|
||||
etype = cfg["type"]
|
||||
name = cfg.get("name", etype)
|
||||
|
||||
if etype == "gummy":
|
||||
from .gummy import GummyEngine
|
||||
|
||||
return GummyEngine(
|
||||
name=name,
|
||||
api_key=cfg["api_key"],
|
||||
model=cfg.get("model", "gummy-realtime-v1"),
|
||||
cost_per_min=cfg.get("cost_per_min"),
|
||||
realtime_factor=cfg.get("realtime_factor", 2.0),
|
||||
)
|
||||
if etype == "whisper":
|
||||
from .whisper import WhisperEngine
|
||||
|
||||
return WhisperEngine(
|
||||
name=name,
|
||||
model_size=cfg.get("model_size", "small"),
|
||||
device=cfg.get("device", "cpu"),
|
||||
compute_type=cfg.get("compute_type", "int8"),
|
||||
model_dir=cfg.get("model_dir"),
|
||||
)
|
||||
if etype == "sensevoice":
|
||||
from .funasr import SenseVoiceEngine
|
||||
|
||||
return SenseVoiceEngine(name=name, device=cfg.get("device", "cpu"), model_dir=cfg.get("model_dir"))
|
||||
if etype == "funasr":
|
||||
from .funasr import ParaformerEngine
|
||||
|
||||
return ParaformerEngine(name=name, device=cfg.get("device", "cpu"), model_dir=cfg.get("model_dir"))
|
||||
|
||||
raise ValueError(f"未知引擎类型: {etype}")
|
||||
|
||||
|
||||
__all__ = ["Engine", "Transcript", "build_engine"]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""引擎接口与单次识别结果。
|
||||
|
||||
两类引擎统一到 transcribe()，但计时口径不同：
|
||||
- streaming（gummy）：有首包延迟 first_partial_sec、定稿延迟 finalize_sec。
|
||||
- offline（本地）：只有总处理时长与 RTF；资源探针在 runner 层包裹（仅本地）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Transcript:
|
||||
text: str
|
||||
audio_sec: float
|
||||
proc_sec: float # transcribe 墙钟耗时
|
||||
first_partial_sec: float | None = None
|
||||
finalize_sec: float | None = None
|
||||
peak_rss_mb: float | None = None # 由 runner 的 ResourceProbe 回填（本地引擎）
|
||||
avg_cpu: float | None = None
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def rtf(self) -> float | None:
|
||||
if not self.audio_sec:
|
||||
return None
|
||||
return self.proc_sec / self.audio_sec
|
||||
|
||||
|
||||
class Engine:
|
||||
"""引擎基类。子类至少实现 transcribe()；load()/unload() 处理重模型生命周期。"""
|
||||
|
||||
name: str = "base"
|
||||
kind: str = "offline" # "offline" | "streaming"
|
||||
is_local: bool = True
|
||||
|
||||
def load(self) -> None:
|
||||
"""加载模型（本地引擎重操作）。云端引擎可空实现。"""
|
||||
|
||||
def transcribe(self, audio_path: str) -> Transcript:
|
||||
raise NotImplementedError
|
||||
|
||||
def unload(self) -> None:
|
||||
"""释放模型，便于多引擎串行评估时回收内存。"""
|
||||
|
||||
def model_size_mb(self) -> float | None:
|
||||
"""磁盘上模型权重大小（MB）；云端返回 None。"""
|
||||
return None
|
||||
|
||||
def cost_per_min(self) -> float | None:
|
||||
"""每分钟成本（云端）；本地返回 None。"""
|
||||
return None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""本地 FunASR 引擎：SenseVoice-Small（多语，中英混说强）与 Paraformer-zh（纯中 SOTA）。
|
||||
|
||||
依赖：pip install '.[funasr]'
|
||||
两者都走 funasr.AutoModel，离线整段识别。SenseVoice 输出带 <|zh|><|EMO|> 等富标签，需剥离。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from ..audio import duration_sec
|
||||
from ..metrics.resource import dir_size_mb
|
||||
from .base import Engine, Transcript
|
||||
|
||||
_TAG = re.compile(r"<\|[^|]*\|>")
|
||||
|
||||
|
||||
def _strip_tags(text: str) -> str:
|
||||
return _TAG.sub("", text).strip()
|
||||
|
||||
|
||||
class _FunASRBase(Engine):
|
||||
kind = "offline"
|
||||
is_local = True
|
||||
repo: str = ""
|
||||
|
||||
def __init__(self, name: str, device: str = "cpu", model_dir: str | None = None):
|
||||
self.name = name
|
||||
self.device = device
|
||||
self.model_dir = model_dir
|
||||
self.model = None
|
||||
self._weights_path: str | None = None
|
||||
|
||||
def load(self) -> None:
|
||||
from funasr import AutoModel
|
||||
|
||||
kwargs = {"model": self.repo, "disable_update": True, "device": self.device}
|
||||
if self.model_dir:
|
||||
kwargs["cache_dir"] = self.model_dir
|
||||
self.model = AutoModel(**kwargs)
|
||||
self._resolve_weights_path()
|
||||
|
||||
def _resolve_weights_path(self) -> None:
|
||||
# funasr 默认从 modelscope 下载；尝试常见属性与缓存目录（best-effort）
|
||||
for attr in ("model_path", "model_pth", "kwargs"):
|
||||
val = getattr(self.model, attr, None)
|
||||
if isinstance(val, str) and os.path.exists(val):
|
||||
self._weights_path = val
|
||||
return
|
||||
if isinstance(val, dict) and isinstance(val.get("model_path"), str):
|
||||
self._weights_path = val["model_path"]
|
||||
return
|
||||
self._weights_path = self.model_dir
|
||||
|
||||
def transcribe(self, audio_path: str) -> Transcript:
|
||||
if self.model is None:
|
||||
self.load()
|
||||
t0 = time.monotonic()
|
||||
res = self.model.generate(input=audio_path)
|
||||
proc = time.monotonic() - t0
|
||||
text = ""
|
||||
if res and isinstance(res, list) and res[0].get("text"):
|
||||
text = _strip_tags(res[0]["text"])
|
||||
return Transcript(text=text, audio_sec=duration_sec(audio_path), proc_sec=proc)
|
||||
|
||||
def model_size_mb(self) -> float | None:
|
||||
size = dir_size_mb(self._weights_path) if self._weights_path else 0.0
|
||||
return size or None
|
||||
|
||||
def unload(self) -> None:
|
||||
self.model = None
|
||||
|
||||
|
||||
class SenseVoiceEngine(_FunASRBase):
|
||||
repo = "iic/SenseVoiceSmall"
|
||||
|
||||
def __init__(self, name: str = "sensevoice", device: str = "cpu", model_dir: str | None = None):
|
||||
super().__init__(name=name, device=device, model_dir=model_dir)
|
||||
|
||||
|
||||
class ParaformerEngine(_FunASRBase):
|
||||
repo = "paraformer-zh"
|
||||
|
||||
def __init__(self, name: str = "paraformer-zh", device: str = "cpu", model_dir: str | None = None):
|
||||
super().__init__(name=name, device=device, model_dir=model_dir)
|
||||
@@ -0,0 +1,180 @@
|
||||
"""云端 gummy 引擎：阿里云百炼 DashScope 流式 ASR（gummy-realtime-v1）。
|
||||
|
||||
协议对照 server/internal/asr/gummy.go：
|
||||
run-task → 等 task-started → 推二进制 PCM(3200B/16k/pcm) → finish-task → 收 task-finished。
|
||||
下行解析 payload.output.transcription(gummy) 或 sentence(paraformer)，
|
||||
sentence_end 兼容 bool 与字符串 "true"/"false"（gummy.go 的 flexBool 怪癖）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import websocket # websocket-client
|
||||
|
||||
from ..audio import load_pcm16
|
||||
from .base import Engine, Transcript
|
||||
|
||||
WS_URL = "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
|
||||
FRAME = 3200 # 1600 samples = 100ms @16k
|
||||
|
||||
|
||||
def _flex_bool(v) -> bool:
|
||||
return v is True or v == "true"
|
||||
|
||||
|
||||
class GummyEngine(Engine):
|
||||
name = "gummy"
|
||||
kind = "streaming"
|
||||
is_local = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str = "gummy",
|
||||
api_key: str = "",
|
||||
model: str = "gummy-realtime-v1",
|
||||
cost_per_min: float | None = None,
|
||||
realtime_factor: float = 2.0,
|
||||
):
|
||||
self.name = name
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self._cost = cost_per_min
|
||||
self.rt = realtime_factor # 推流倍速；2.0 = 2x 实时（同 gummycheck）
|
||||
|
||||
def cost_per_min(self) -> float | None:
|
||||
return self._cost
|
||||
|
||||
def _run_task_msg(self, task_id: str, sample_rate: int) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"header": {"action": "run-task", "task_id": task_id, "streaming": "duplex"},
|
||||
"payload": {
|
||||
"task_group": "audio",
|
||||
"task": "asr",
|
||||
"function": "recognition",
|
||||
"model": self.model,
|
||||
"parameters": {
|
||||
"sample_rate": sample_rate,
|
||||
"format": "pcm",
|
||||
"transcription_enabled": True,
|
||||
"translation_enabled": False,
|
||||
},
|
||||
"input": {},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
def _finish_task_msg(self, task_id: str) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"header": {"action": "finish-task", "task_id": task_id, "streaming": "duplex"},
|
||||
"payload": {"input": {}},
|
||||
}
|
||||
)
|
||||
|
||||
def transcribe(self, audio_path: str) -> Transcript:
|
||||
pcm, sr = load_pcm16(audio_path, 16000)
|
||||
audio = pcm.tobytes()
|
||||
audio_sec = len(pcm) / sr
|
||||
task_id = uuid.uuid4().hex
|
||||
|
||||
started = threading.Event()
|
||||
finished = threading.Event()
|
||||
results: list[tuple[float, str, bool]] = [] # (rel_t, text, is_final)
|
||||
err: list[str | None] = [None]
|
||||
first_partial: list[float | None] = [None]
|
||||
t0 = time.monotonic()
|
||||
|
||||
try:
|
||||
ws = websocket.create_connection(
|
||||
WS_URL,
|
||||
header=[
|
||||
f"Authorization: bearer {self.api_key}",
|
||||
"X-DashScope-DataInspection: enable",
|
||||
],
|
||||
timeout=20,
|
||||
)
|
||||
except Exception as ex: # 连接失败
|
||||
return Transcript(text="", audio_sec=audio_sec, proc_sec=time.monotonic() - t0, error=f"dial: {ex}")
|
||||
|
||||
def reader() -> None:
|
||||
try:
|
||||
while True:
|
||||
msg = ws.recv()
|
||||
if not msg:
|
||||
continue
|
||||
ev = json.loads(msg)
|
||||
h = ev.get("header", {})
|
||||
event = h.get("event")
|
||||
if event == "task-started":
|
||||
started.set()
|
||||
elif event == "result-generated":
|
||||
out = ev.get("payload", {}).get("output", {})
|
||||
sen = out.get("transcription") or out.get("sentence")
|
||||
if not sen or not sen.get("text"):
|
||||
continue
|
||||
is_final = _flex_bool(sen.get("sentence_end")) or _flex_bool(sen.get("is_sentence_end"))
|
||||
if first_partial[0] is None:
|
||||
first_partial[0] = time.monotonic() - t0
|
||||
results.append((time.monotonic() - t0, sen["text"], is_final))
|
||||
elif event == "task-finished":
|
||||
finished.set()
|
||||
break
|
||||
elif event == "task-failed":
|
||||
err[0] = h.get("error_message", "task-failed")
|
||||
finished.set()
|
||||
break
|
||||
except Exception as ex:
|
||||
if not finished.is_set():
|
||||
err[0] = str(ex)
|
||||
finished.set()
|
||||
|
||||
rt = threading.Thread(target=reader, daemon=True)
|
||||
rt.start()
|
||||
|
||||
try:
|
||||
ws.send(self._run_task_msg(task_id, sr))
|
||||
if not started.wait(timeout=10):
|
||||
ws.close()
|
||||
return Transcript(
|
||||
text="", audio_sec=audio_sec, proc_sec=time.monotonic() - t0, error="task-started timeout"
|
||||
)
|
||||
|
||||
# 推流：每帧 100ms 音频，按倍速 sleep（2x → 50ms）
|
||||
per_frame_sleep = 0.1 / self.rt
|
||||
for off in range(0, len(audio), FRAME):
|
||||
ws.send_binary(audio[off : off + FRAME])
|
||||
time.sleep(per_frame_sleep)
|
||||
last_audio_t = time.monotonic()
|
||||
|
||||
ws.send(self._finish_task_msg(task_id))
|
||||
finished.wait(timeout=30)
|
||||
except Exception as ex:
|
||||
err[0] = err[0] or str(ex)
|
||||
finally:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# gummy 按句给 final；拼接所有 final，无 final 则回退最后一条 partial
|
||||
finals = [t for _, t, f in results if f]
|
||||
text = "".join(finals) if finals else (results[-1][1] if results else "")
|
||||
|
||||
finalize_sec = None
|
||||
final_rel_times = [rel for rel, _, f in results if f]
|
||||
if final_rel_times:
|
||||
finalize_sec = max(0.0, (t0 + final_rel_times[-1]) - last_audio_t)
|
||||
|
||||
return Transcript(
|
||||
text=text,
|
||||
audio_sec=audio_sec,
|
||||
proc_sec=time.monotonic() - t0,
|
||||
first_partial_sec=first_partial[0],
|
||||
finalize_sec=finalize_sec,
|
||||
error=err[0],
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""本地 faster-whisper 引擎（CTranslate2，CPU-first）。多语，对中英混说有基础能力。
|
||||
|
||||
依赖：pip install '.[whisper]'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from ..metrics.resource import dir_size_mb
|
||||
from .base import Engine, Transcript
|
||||
|
||||
|
||||
class WhisperEngine(Engine):
|
||||
kind = "offline"
|
||||
is_local = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str = "whisper-small",
|
||||
model_size: str = "small",
|
||||
device: str = "cpu",
|
||||
compute_type: str = "int8",
|
||||
model_dir: str | None = None,
|
||||
):
|
||||
self.name = name
|
||||
self.model_size = model_size
|
||||
self.device = device
|
||||
self.compute_type = compute_type
|
||||
self.model_dir = model_dir
|
||||
self.model = None
|
||||
self._weights_path: str | None = None
|
||||
|
||||
def load(self) -> None:
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
self.model = WhisperModel(
|
||||
self.model_size,
|
||||
device=self.device,
|
||||
compute_type=self.compute_type,
|
||||
download_root=self.model_dir,
|
||||
)
|
||||
self._resolve_weights_path()
|
||||
|
||||
def _resolve_weights_path(self) -> None:
|
||||
# 解析磁盘权重路径用于报模型大小（best-effort）
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
repo = f"Systran/faster-whisper-{self.model_size}"
|
||||
self._weights_path = snapshot_download(repo, local_files_only=True, cache_dir=self.model_dir)
|
||||
except Exception:
|
||||
self._weights_path = self.model_dir
|
||||
|
||||
def transcribe(self, audio_path: str) -> Transcript:
|
||||
if self.model is None:
|
||||
self.load()
|
||||
t0 = time.monotonic()
|
||||
# language=None → 自动检测；中英混说交给模型自身
|
||||
segments, info = self.model.transcribe(audio_path, language=None, beam_size=5)
|
||||
text = "".join(seg.text for seg in segments) # 生成器在此 join 时才真正解码
|
||||
proc = time.monotonic() - t0
|
||||
return Transcript(text=text.strip(), audio_sec=float(info.duration), proc_sec=proc)
|
||||
|
||||
def model_size_mb(self) -> float | None:
|
||||
size = dir_size_mb(self._weights_path) if self._weights_path else 0.0
|
||||
return size or None
|
||||
|
||||
def unload(self) -> None:
|
||||
self.model = None
|
||||
Reference in New Issue
Block a user