feat(eval): ASR 模型评估框架
横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer)， 重点覆盖中英混说，产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,98 @@
|
||||
"""准确率指标：CER（字级）/ WER（词级）/ MER（中英混合）。
|
||||
|
||||
分词策略：
|
||||
- CER : 去空格后逐字符（中文每字、英文每字母各一 token）。
|
||||
- WER : 按空白切词（主要对英文有意义）。
|
||||
- MER : 中文按【字】、英文/数字按【词】，混合 token 序列算编辑距离——这是中英混说的 headline 指标。
|
||||
|
||||
聚合：错误率必须在语料级聚合（累加 S/D/I/N 后再求率），不能对每条样本的率求平均，
|
||||
否则短句权重被放大、结果失真。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import jiwer
|
||||
|
||||
from .text_norm import normalize
|
||||
|
||||
_CJK = re.compile(r"[一-鿿㐀-䶿]")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Counts:
|
||||
"""一组编辑距离统计。N = 参考长度 = S + D + H。"""
|
||||
|
||||
S: int = 0 # 替换
|
||||
D: int = 0 # 删除
|
||||
I: int = 0 # 插入
|
||||
N: int = 0 # 参考 token 数
|
||||
|
||||
def __add__(self, o: "Counts") -> "Counts":
|
||||
return Counts(self.S + o.S, self.D + o.D, self.I + o.I, self.N + o.N)
|
||||
|
||||
@property
|
||||
def rate(self) -> float:
|
||||
if self.N == 0:
|
||||
return 1.0 if (self.S + self.D + self.I) > 0 else 0.0
|
||||
return (self.S + self.D + self.I) / self.N
|
||||
|
||||
|
||||
def char_tokens(text: str) -> list[str]:
|
||||
return [c for c in text if not c.isspace()]
|
||||
|
||||
|
||||
def word_tokens(text: str) -> list[str]:
|
||||
return text.split()
|
||||
|
||||
|
||||
def mixed_tokens(text: str) -> list[str]:
|
||||
"""中文按字、其余按词。"""
|
||||
tokens: list[str] = []
|
||||
buf = ""
|
||||
for ch in text:
|
||||
if _CJK.match(ch):
|
||||
if buf.strip():
|
||||
tokens.extend(buf.split())
|
||||
buf = ""
|
||||
tokens.append(ch)
|
||||
else:
|
||||
buf += ch
|
||||
if buf.strip():
|
||||
tokens.extend(buf.split())
|
||||
return tokens
|
||||
|
||||
|
||||
def _counts(ref_tokens: list[str], hyp_tokens: list[str]) -> Counts:
|
||||
if not ref_tokens:
|
||||
return Counts(S=0, D=0, I=len(hyp_tokens), N=0)
|
||||
out = jiwer.process_words(" ".join(ref_tokens), " ".join(hyp_tokens))
|
||||
return Counts(
|
||||
S=out.substitutions,
|
||||
D=out.deletions,
|
||||
I=out.insertions,
|
||||
N=out.substitutions + out.deletions + out.hits,
|
||||
)
|
||||
|
||||
|
||||
def score_sample(ref_text: str, hyp_text: str) -> dict[str, Counts]:
|
||||
"""对一条样本算 CER/WER/MER 的原始计数（供语料级聚合）。"""
|
||||
ref = normalize(ref_text)
|
||||
hyp = normalize(hyp_text)
|
||||
return {
|
||||
"cer": _counts(char_tokens(ref), char_tokens(hyp)),
|
||||
"wer": _counts(word_tokens(ref), word_tokens(hyp)),
|
||||
"mer": _counts(mixed_tokens(ref), mixed_tokens(hyp)),
|
||||
}
|
||||
|
||||
|
||||
def empty_counts() -> dict[str, Counts]:
|
||||
return {"cer": Counts(), "wer": Counts(), "mer": Counts()}
|
||||
|
||||
|
||||
def add_counts(acc: dict[str, Counts], one: dict[str, Counts]) -> dict[str, Counts]:
|
||||
for k in acc:
|
||||
acc[k] = acc[k] + one[k]
|
||||
return acc
|
||||
@@ -0,0 +1,85 @@
|
||||
"""资源探针：本地引擎推理时采样峰值内存/CPU；模型大小读磁盘。
|
||||
|
||||
注意：psutil 测的是整个 Python 进程 RSS（含解释器与已加载库），用于本地模型横向对比是
|
||||
合理代理量；报告会标注硬件环境，且只在本地引擎启用，云端 gummy 不测（填 N/A）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import psutil
|
||||
|
||||
|
||||
class ResourceProbe:
|
||||
"""上下文管理器：with 块内后台采样进程 RSS / CPU%。"""
|
||||
|
||||
def __init__(self, interval: float = 0.05):
|
||||
self.interval = interval
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._proc = psutil.Process(os.getpid())
|
||||
self.peak_rss = 0
|
||||
self._cpu: list[float] = []
|
||||
|
||||
def __enter__(self) -> "ResourceProbe":
|
||||
self._proc.cpu_percent(None) # 预热：首次调用返回 0，丢弃
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
self.peak_rss = max(self.peak_rss, self._proc.memory_info().rss)
|
||||
c = self._proc.cpu_percent(None)
|
||||
if c > 0:
|
||||
self._cpu.append(c)
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(self.interval)
|
||||
|
||||
def __exit__(self, *_exc) -> None:
|
||||
self._stop.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=1.0)
|
||||
|
||||
@property
|
||||
def peak_rss_mb(self) -> float:
|
||||
return self.peak_rss / 1024 / 1024
|
||||
|
||||
@property
|
||||
def avg_cpu(self) -> float:
|
||||
return sum(self._cpu) / len(self._cpu) if self._cpu else 0.0
|
||||
|
||||
|
||||
def dir_size_mb(path: str) -> float:
|
||||
"""目录下所有文件体积之和（MB）；路径不存在返回 0。"""
|
||||
total = 0
|
||||
if not path or not os.path.exists(path):
|
||||
return 0.0
|
||||
if os.path.isfile(path):
|
||||
return os.path.getsize(path) / 1024 / 1024
|
||||
for root, _dirs, files in os.walk(path):
|
||||
for f in files:
|
||||
try:
|
||||
total += os.path.getsize(os.path.join(root, f))
|
||||
except OSError:
|
||||
pass
|
||||
return total / 1024 / 1024
|
||||
|
||||
|
||||
def hardware_info() -> dict:
|
||||
"""报告里标注的硬件环境。"""
|
||||
import platform
|
||||
|
||||
vm = psutil.virtual_memory()
|
||||
return {
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"cpu_count": psutil.cpu_count(logical=True),
|
||||
"cpu_count_physical": psutil.cpu_count(logical=False),
|
||||
"total_mem_gb": round(vm.total / 1024**3, 1),
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
"""评分前文本归一化。
|
||||
|
||||
中英混说不归一化就算分会严重失真（标点、繁简、全半角、大小写都会被算成错）。
|
||||
本模块做最小必要归一化；繁→简依赖 opencc，缺失时自动降级（仅告警一次）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
_cc = None
|
||||
_cc_warned = False
|
||||
|
||||
|
||||
def _converter():
|
||||
global _cc, _cc_warned
|
||||
if _cc is not None:
|
||||
return _cc
|
||||
try:
|
||||
from opencc import OpenCC
|
||||
|
||||
_cc = OpenCC("t2s")
|
||||
except Exception:
|
||||
_cc = False # 标记尝试过且失败
|
||||
if not _cc_warned:
|
||||
print("[text_norm] 警告：opencc 不可用，跳过繁→简归一化", file=sys.stderr)
|
||||
_cc_warned = True
|
||||
return _cc
|
||||
|
||||
|
||||
# 中英文常见标点（评分时整体移除）
|
||||
_PUNCT = set(
|
||||
"，。！？、；：“”‘’（）《》【】〔〕…—～·.,!?;:\"'()<>[]{}~`@#$%^&*-_=+|\\/。．"
|
||||
)
|
||||
_WS = re.compile(r"\s+")
|
||||
|
||||
|
||||
def normalize(text: str, *, t2s: bool = True, lower: bool = True) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
# 全角→半角 + Unicode 兼容分解（NFKC 会把 ＡＢＣ→ABC、１２３→123）
|
||||
text = unicodedata.normalize("NFKC", text)
|
||||
if t2s:
|
||||
cc = _converter()
|
||||
if cc:
|
||||
text = cc.convert(text)
|
||||
if lower:
|
||||
text = text.lower()
|
||||
text = "".join(ch for ch in text if ch not in _PUNCT)
|
||||
text = _WS.sub(" ", text).strip()
|
||||
return text
|
||||
Reference in New Issue
Block a user