dudu MVP：五端语音输入法初始提交
- server：Go 网关（WS 流式识别中继/计费配额/微信登录支付 mock/反馈/埋点），gummy provider 已真实联调 - desktop：Tauri 2（全局快捷键 push-to-talk/浮层/托盘/设置/登录购买/反馈/首启引导） - android：Compose 主 App + IME（键盘内录音直传） - ios：App + 键盘扩展（1A spike 实证键盘内不可录音，走 deep link 听写） - design/design-pipeline：设计系统 + token 导出 iOS/Android 主题 - doc：前后端设计文档（HTML）；web：官网宣传页；todo：任务看板 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,221 @@
|
||||
// gummy provider：阿里云百炼 DashScope 流式语音识别（gummy-realtime-v1）。
|
||||
// WS 协议：run-task → 推二进制音频 → finish-task；下行 result-generated / task-finished / task-failed。
|
||||
// ⚠️ 待真实 DASHSCOPE_API_KEY 到位后联调验证（6B 验收项），协议细节以
|
||||
// https://help.aliyun.com/zh/model-studio/ 实时语音识别文档为准。
|
||||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
const dashscopeWS = "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
|
||||
|
||||
type GummyProvider struct {
|
||||
APIKey string
|
||||
Model string // 默认 gummy-realtime-v1
|
||||
}
|
||||
|
||||
func NewGummy(apiKey string) *GummyProvider {
|
||||
return &GummyProvider{APIKey: apiKey, Model: "gummy-realtime-v1"}
|
||||
}
|
||||
|
||||
func (g *GummyProvider) Name() string { return "gummy" }
|
||||
|
||||
func (g *GummyProvider) StartSession(ctx context.Context, cfg SessionConfig) (Session, error) {
|
||||
header := map[string][]string{
|
||||
"Authorization": {"bearer " + g.APIKey},
|
||||
"X-DashScope-DataInspection": {"enable"},
|
||||
}
|
||||
conn, _, err := websocket.DefaultDialer.DialContext(ctx, dashscopeWS, header)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dashscope dial: %w", err)
|
||||
}
|
||||
|
||||
taskID := uuid.NewString()
|
||||
runTask := map[string]any{
|
||||
"header": map[string]any{
|
||||
"action": "run-task",
|
||||
"task_id": taskID,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": map[string]any{
|
||||
"task_group": "audio",
|
||||
"task": "asr",
|
||||
"function": "recognition",
|
||||
"model": g.Model,
|
||||
"parameters": map[string]any{
|
||||
"sample_rate": cfg.SampleRate,
|
||||
"format": "pcm",
|
||||
"transcription_enabled": true,
|
||||
"translation_enabled": false,
|
||||
},
|
||||
"input": map[string]any{},
|
||||
},
|
||||
}
|
||||
if err := conn.WriteJSON(runTask); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("dashscope run-task: %w", err)
|
||||
}
|
||||
|
||||
s := &gummySession{
|
||||
conn: conn,
|
||||
taskID: taskID,
|
||||
results: make(chan Result, 64),
|
||||
done: make(chan struct{}),
|
||||
started: make(chan struct{}),
|
||||
}
|
||||
go s.readLoop()
|
||||
|
||||
// 协议要求：必须等 task-started 后才能推音频，否则服务端静默丢弃
|
||||
select {
|
||||
case <-s.started:
|
||||
return s, nil
|
||||
case <-s.done:
|
||||
return nil, fmt.Errorf("dashscope: closed before task-started")
|
||||
case <-ctx.Done():
|
||||
_ = conn.Close()
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(10 * time.Second):
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("dashscope: task-started timeout")
|
||||
}
|
||||
}
|
||||
|
||||
type gummySession struct {
|
||||
conn *websocket.Conn
|
||||
taskID string
|
||||
results chan Result
|
||||
done chan struct{}
|
||||
started chan struct{} // task-started 已收到
|
||||
writeMu sync.Mutex
|
||||
closeOnce sync.Once
|
||||
startOnce sync.Once
|
||||
lastText string // 当前句已下发文本，用于切分 partial/final
|
||||
}
|
||||
|
||||
func (s *gummySession) SendAudio(pcm []byte) error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
select {
|
||||
case <-s.done:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
return s.conn.WriteMessage(websocket.BinaryMessage, pcm)
|
||||
}
|
||||
|
||||
// dashscope 下行事件结构（按需字段）。
|
||||
type dsEvent struct {
|
||||
Header struct {
|
||||
Event string `json:"event"` // task-started | result-generated | task-finished | task-failed
|
||||
ErrorMessage string `json:"error_message"`
|
||||
} `json:"header"`
|
||||
Payload struct {
|
||||
Output struct {
|
||||
// gummy: transcription；paraformer: sentence —— 两种都尝试解析
|
||||
Transcription *dsSentence `json:"transcription"`
|
||||
Sentence *dsSentence `json:"sentence"`
|
||||
} `json:"output"`
|
||||
} `json:"payload"`
|
||||
}
|
||||
|
||||
type dsSentence struct {
|
||||
Text string `json:"text"`
|
||||
SentenceEnd flexBool `json:"sentence_end"`
|
||||
IsSentenceEnd flexBool `json:"is_sentence_end"`
|
||||
BeginTime int64 `json:"begin_time"`
|
||||
EndTime int64 `json:"end_time"`
|
||||
}
|
||||
|
||||
// flexBool 兼容 dashscope 同一字段混用 bool 与字符串（"true"/"false"）的情况。
|
||||
type flexBool bool
|
||||
|
||||
func (b *flexBool) UnmarshalJSON(data []byte) error {
|
||||
switch string(data) {
|
||||
case "true", `"true"`:
|
||||
*b = true
|
||||
default:
|
||||
*b = false
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *gummySession) readLoop() {
|
||||
defer func() {
|
||||
s.closeOnce.Do(func() { close(s.done) })
|
||||
close(s.results)
|
||||
s.conn.Close()
|
||||
}()
|
||||
for {
|
||||
_, data, err := s.conn.ReadMessage()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-s.done: // 主动关闭
|
||||
default:
|
||||
s.results <- Result{Err: err}
|
||||
}
|
||||
return
|
||||
}
|
||||
if os.Getenv("GUMMY_DEBUG") != "" {
|
||||
fmt.Fprintf(os.Stderr, "[gummy] %s\n", data)
|
||||
}
|
||||
var ev dsEvent
|
||||
if json.Unmarshal(data, &ev) != nil {
|
||||
continue
|
||||
}
|
||||
switch ev.Header.Event {
|
||||
case "task-started":
|
||||
s.startOnce.Do(func() { close(s.started) })
|
||||
case "result-generated":
|
||||
sen := ev.Payload.Output.Transcription
|
||||
if sen == nil {
|
||||
sen = ev.Payload.Output.Sentence
|
||||
}
|
||||
if sen == nil || sen.Text == "" {
|
||||
continue
|
||||
}
|
||||
isFinal := bool(sen.SentenceEnd) || bool(sen.IsSentenceEnd)
|
||||
s.results <- Result{Text: sen.Text, IsFinal: isFinal, EndTimeMs: sen.EndTime}
|
||||
if isFinal {
|
||||
s.lastText = ""
|
||||
} else {
|
||||
s.lastText = sen.Text
|
||||
}
|
||||
case "task-finished":
|
||||
return
|
||||
case "task-failed":
|
||||
s.results <- Result{Err: fmt.Errorf("dashscope: %s", ev.Header.ErrorMessage)}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *gummySession) Results() <-chan Result { return s.results }
|
||||
|
||||
// Close 发 finish-task 让上游 flush 尾部，等 task-finished 后 readLoop 退出。
|
||||
func (s *gummySession) Close() error {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
select {
|
||||
case <-s.done:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
finish := map[string]any{
|
||||
"header": map[string]any{
|
||||
"action": "finish-task",
|
||||
"task_id": s.taskID,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": map[string]any{"input": map[string]any{}},
|
||||
}
|
||||
return s.conn.WriteJSON(finish)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestGummyLive 9A gummy 实测（需真实凭证，无则跳过）：
|
||||
//
|
||||
// rbw get dashscope-api-key | { read -r K; DASHSCOPE_API_KEY="$K" GUMMY_TEST_WAV=/path/16k.wav go test ./internal/asr -run TestGummyLive -v; }
|
||||
func TestGummyLive(t *testing.T) {
|
||||
key := os.Getenv("DASHSCOPE_API_KEY")
|
||||
wav := os.Getenv("GUMMY_TEST_WAV")
|
||||
if key == "" || wav == "" {
|
||||
t.Skip("DASHSCOPE_API_KEY / GUMMY_TEST_WAV 未设置，跳过 gummy 实测")
|
||||
}
|
||||
pcm, err := readWavDataT(t, wav)
|
||||
if err != nil {
|
||||
t.Fatalf("读 wav: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
sess, err := NewGummy(key).StartSession(ctx, SessionConfig{SampleRate: 16000, SessionID: "live-test"})
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
|
||||
var partials, finals int
|
||||
var lastFinal string
|
||||
var maxEndMs int64
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for r := range sess.Results() {
|
||||
if r.Err != nil {
|
||||
t.Errorf("result err: %v", r.Err)
|
||||
return
|
||||
}
|
||||
if r.IsFinal {
|
||||
finals++
|
||||
lastFinal = r.Text
|
||||
} else {
|
||||
partials++
|
||||
}
|
||||
if r.EndTimeMs > maxEndMs {
|
||||
maxEndMs = r.EndTimeMs
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for off := 0; off < len(pcm); off += 3200 {
|
||||
end := min(off+3200, len(pcm))
|
||||
if err := sess.SendAudio(pcm[off:end]); err != nil {
|
||||
t.Fatalf("SendAudio: %v", err)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
_ = sess.Close()
|
||||
<-done
|
||||
|
||||
if partials == 0 || finals == 0 {
|
||||
t.Fatalf("期望 partial>0 且 final>0，实得 partial=%d final=%d", partials, finals)
|
||||
}
|
||||
if strings.TrimSpace(lastFinal) == "" {
|
||||
t.Fatal("final 文本为空")
|
||||
}
|
||||
audioMs := int64(len(pcm)) * 1000 / 32000
|
||||
if maxEndMs <= 0 || maxEndMs > audioMs+3000 {
|
||||
t.Fatalf("provider 时间戳异常: end=%dms audio=%dms", maxEndMs, audioMs)
|
||||
}
|
||||
t.Logf("partials=%d finals=%d end=%dms final=%q", partials, finals, maxEndMs, lastFinal)
|
||||
}
|
||||
|
||||
func readWavDataT(t *testing.T, path string) ([]byte, error) {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off := 12
|
||||
for off+8 <= len(b) {
|
||||
id := string(b[off : off+4])
|
||||
size := int(binary.LittleEndian.Uint32(b[off+4 : off+8]))
|
||||
if id == "data" {
|
||||
return b[off+8 : min(off+8+size, len(b))], nil
|
||||
}
|
||||
off += 8 + size
|
||||
}
|
||||
return nil, os.ErrInvalid
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"dudu/server/pkg/protocol"
|
||||
)
|
||||
|
||||
// MockProvider 测试/开发用：按收到的音频量逐字"识别"出预置句子，
|
||||
// 行为模拟 Gummy：每帧产出 partial，句读处产出 final。
|
||||
type MockProvider struct {
|
||||
// Script 轮换使用的识别文本
|
||||
Script []string
|
||||
mu sync.Mutex
|
||||
idx int
|
||||
}
|
||||
|
||||
func NewMock() *MockProvider {
|
||||
return &MockProvider{Script: []string{
|
||||
"帮我把这份周报整理一下，重点突出本周的进展。",
|
||||
"好的，我马上把文件发给你，大概十分钟之内。",
|
||||
}}
|
||||
}
|
||||
|
||||
func (m *MockProvider) Name() string { return "mock" }
|
||||
|
||||
func (m *MockProvider) StartSession(ctx context.Context, cfg SessionConfig) (Session, error) {
|
||||
m.mu.Lock()
|
||||
sentence := []rune(m.Script[m.idx%len(m.Script)])
|
||||
m.idx++
|
||||
m.mu.Unlock()
|
||||
s := &mockSession{
|
||||
sentence: sentence,
|
||||
results: make(chan Result, 64),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type mockSession struct {
|
||||
sentence []rune
|
||||
results chan Result
|
||||
pos int // 已"识别"的字数
|
||||
committed int // 已 final 的字数
|
||||
bytes int
|
||||
closeOnce sync.Once
|
||||
done chan struct{}
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// 每帧（100ms 音频）识别出 1 个字；遇句读（，。）将其前内容定稿为 final。
|
||||
func (s *mockSession) SendAudio(pcm []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
select {
|
||||
case <-s.done:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
s.bytes += len(pcm)
|
||||
for s.pos < len(s.sentence) && s.bytes >= (s.pos+1)*protocol.FrameBytes {
|
||||
s.pos++
|
||||
ch := s.sentence[s.pos-1]
|
||||
endMs := int64(s.bytes) * 1000 / 32000 // 模拟 provider 侧时间戳=实收音频时长
|
||||
if ch == '，' || ch == '。' {
|
||||
s.emit(Result{Text: string(s.sentence[s.committed:s.pos]), IsFinal: true, EndTimeMs: endMs})
|
||||
s.committed = s.pos
|
||||
} else {
|
||||
s.emit(Result{Text: string(s.sentence[s.committed:s.pos]), EndTimeMs: endMs})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mockSession) emit(r Result) {
|
||||
select {
|
||||
case s.results <- r:
|
||||
default: // 测试场景下不阻塞
|
||||
}
|
||||
}
|
||||
|
||||
func (s *mockSession) Results() <-chan Result { return s.results }
|
||||
|
||||
// Close flush 未定稿部分为 final 并关闭通道。
|
||||
func (s *mockSession) Close() error {
|
||||
s.closeOnce.Do(func() {
|
||||
s.mu.Lock()
|
||||
if s.pos > s.committed {
|
||||
s.emit(Result{Text: string(s.sentence[s.committed:s.pos]), IsFinal: true, EndTimeMs: int64(s.bytes) * 1000 / 32000})
|
||||
s.committed = s.pos
|
||||
}
|
||||
close(s.done)
|
||||
close(s.results)
|
||||
s.mu.Unlock()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"dudu/server/pkg/protocol"
|
||||
)
|
||||
|
||||
func TestMockProviderStreams(t *testing.T) {
|
||||
p := NewMock()
|
||||
p.Script = []string{"你好，世界。"}
|
||||
s, err := p.StartSession(context.Background(), SessionConfig{SampleRate: 16000})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frame := make([]byte, protocol.FrameBytes)
|
||||
for i := 0; i < 10; i++ { // 1s 音频 = 10 帧 → 全句 6 字识别完
|
||||
if err := s.SendAudio(frame); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var finals []string
|
||||
partials := 0
|
||||
for r := range s.Results() {
|
||||
if r.IsFinal {
|
||||
finals = append(finals, r.Text)
|
||||
} else {
|
||||
partials++
|
||||
}
|
||||
}
|
||||
got := strings.Join(finals, "")
|
||||
if got != "你好，世界。" {
|
||||
t.Fatalf("final text mismatch: %q (finals=%v)", got, finals)
|
||||
}
|
||||
if partials == 0 {
|
||||
t.Fatal("expect partial results before finals")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Package asr 流式识别 Provider 抽象：网关只依赖此接口，gummy / volcano / mock 可切换。
|
||||
package asr
|
||||
|
||||
import "context"
|
||||
|
||||
type SessionConfig struct {
|
||||
SampleRate int
|
||||
// SessionID 仅用于日志关联
|
||||
SessionID string
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Text string
|
||||
IsFinal bool
|
||||
// EndTimeMs Provider 报告的本句结束时间（相对会话起点，毫秒）；0=未提供。
|
||||
// 网关用它与实收帧累计时长做计量交叉校验。
|
||||
EndTimeMs int64
|
||||
// Err 非空表示会话级错误，随后 Results 通道关闭
|
||||
Err error
|
||||
}
|
||||
|
||||
type Session interface {
|
||||
// SendAudio 推一帧 PCM（100ms / 3200B）
|
||||
SendAudio(pcm []byte) error
|
||||
// Results 识别结果通道；会话结束（Close 或上游完成）后关闭
|
||||
Results() <-chan Result
|
||||
// Close 结束会话：要求上游 flush 尾部音频并下发最终 final 后关闭通道
|
||||
Close() error
|
||||
}
|
||||
|
||||
type Provider interface {
|
||||
Name() string
|
||||
StartSession(ctx context.Context, cfg SessionConfig) (Session, error)
|
||||
}
|
||||
Reference in New Issue
Block a user