feat(eval): ASR 模型评估框架
横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer), 重点覆盖中英混说,产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
"""引擎接口与单次识别结果。
|
||||
|
||||
两类引擎统一到 transcribe(),但计时口径不同:
|
||||
- streaming(gummy):有首包延迟 first_partial_sec、定稿延迟 finalize_sec。
|
||||
- offline(本地):只有总处理时长与 RTF;资源探针在 runner 层包裹(仅本地)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Transcript:
|
||||
text: str
|
||||
audio_sec: float
|
||||
proc_sec: float # transcribe 墙钟耗时
|
||||
first_partial_sec: float | None = None
|
||||
finalize_sec: float | None = None
|
||||
peak_rss_mb: float | None = None # 由 runner 的 ResourceProbe 回填(本地引擎)
|
||||
avg_cpu: float | None = None
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def rtf(self) -> float | None:
|
||||
if not self.audio_sec:
|
||||
return None
|
||||
return self.proc_sec / self.audio_sec
|
||||
|
||||
|
||||
class Engine:
|
||||
"""引擎基类。子类至少实现 transcribe();load()/unload() 处理重模型生命周期。"""
|
||||
|
||||
name: str = "base"
|
||||
kind: str = "offline" # "offline" | "streaming"
|
||||
is_local: bool = True
|
||||
|
||||
def load(self) -> None:
|
||||
"""加载模型(本地引擎重操作)。云端引擎可空实现。"""
|
||||
|
||||
def transcribe(self, audio_path: str) -> Transcript:
|
||||
raise NotImplementedError
|
||||
|
||||
def unload(self) -> None:
|
||||
"""释放模型,便于多引擎串行评估时回收内存。"""
|
||||
|
||||
def model_size_mb(self) -> float | None:
|
||||
"""磁盘上模型权重大小(MB);云端返回 None。"""
|
||||
return None
|
||||
|
||||
def cost_per_min(self) -> float | None:
|
||||
"""每分钟成本(云端);本地返回 None。"""
|
||||
return None
|
||||
Reference in New Issue
Block a user