wangjia
25acf9db6e
横向对比云端 gummy 与本地开源模型(faster-whisper/SenseVoice/Paraformer)， 重点覆盖中英混说，产出准确率(CER/WER/MER)/速度(延迟/RTF)/资源(cpu/mem/模型大小) 对比报告。公共集(ASCEND/AISHELL/LibriSpeech)统一走 HF 适配器 + 自定义 JSONL manifest。 gummy 引擎对照 server/internal/asr/gummy.go 协议移植。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
"""自定义集：JSONL manifest，每行一条样本。
|
|
|
|
格式（audio 路径相对 manifest 文件所在目录解析）：
|
|
{"audio": "clips/a.wav", "text": "帮我把这份 weekly report 整理一下", "lang": "zh-en", "domain": "tech"}
|
|
|
|
lang 缺省按文本自动猜（含中文且含 ASCII 字母→zh-en；纯 ASCII→en；否则 zh）。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
from collections.abc import Iterator
|
|
|
|
from ..manifest import Sample
|
|
|
|
_CJK = re.compile(r"[一-鿿]")
|
|
_ASCII_ALPHA = re.compile(r"[A-Za-z]")
|
|
|
|
|
|
def _guess_lang(text: str) -> str:
|
|
has_cjk = bool(_CJK.search(text))
|
|
has_en = bool(_ASCII_ALPHA.search(text))
|
|
if has_cjk and has_en:
|
|
return "zh-en"
|
|
if has_en and not has_cjk:
|
|
return "en"
|
|
return "zh"
|
|
|
|
|
|
class CustomDataset:
|
|
def __init__(self, cfg: dict):
|
|
self.name = cfg.get("name", "custom")
|
|
self.manifest = cfg["manifest"]
|
|
self.domain_default = cfg.get("domain", "general")
|
|
self._base = os.path.dirname(os.path.abspath(self.manifest))
|
|
|
|
def samples(self, limit: int | None = None) -> Iterator[Sample]:
|
|
with open(self.manifest, encoding="utf-8") as f:
|
|
n = 0
|
|
for i, line in enumerate(f):
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if limit is not None and n >= limit:
|
|
break
|
|
row = json.loads(line)
|
|
text = (row.get("text") or "").strip()
|
|
audio = row["audio"]
|
|
if not os.path.isabs(audio):
|
|
audio = os.path.join(self._base, audio)
|
|
lang = row.get("lang") or _guess_lang(text)
|
|
n += 1
|
|
yield Sample(
|
|
id=row.get("id", f"{self.name}-{i:06d}"),
|
|
audio_path=audio,
|
|
ref_text=text,
|
|
lang=lang,
|
|
domain=row.get("domain", self.domain_default),
|
|
dataset=self.name,
|
|
)
|