feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗

盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通)
而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。

控制面:
- agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖),
  经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。
- NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。
- effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode
  统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。

客户端(connection_provider 连通看门狗):
- 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。
- 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。
- 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。

测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗
智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-29 13:19:22 +08:00
parent 6249d5b2ea
commit 2080062d8c
15 changed files with 448 additions and 29 deletions
+1
View File
@@ -183,6 +183,7 @@ func (h *Handler) Heartbeat(ctx context.Context, req *agentv1.HeartbeatRequest)
BandwidthUpBps: req.BandwidthUpBps,
BandwidthDownBps: req.BandwidthDownBps,
CPUPercent: req.CPUPercent,
DataPlaneHealthy: req.DataPlaneHealthy,
}); err != nil {
slog.Warn("nodes.Handler.Heartbeat: load write failed",
"node_uuid", nodeUUID, "err", err)
+15 -1
View File
@@ -24,6 +24,10 @@ type NodeLoad struct {
BandwidthUpBps int64
BandwidthDownBps int64
CPUPercent float64
// DataPlaneHealthy is the agent's last sing-box health probe result. The
// control plane downgrades a node to "down" when this is false (even if the
// agent's gRPC stream is still up). Absent in older hashes → read as true.
DataPlaneHealthy bool
}
// LoadCache reads and writes node:load Redis hashes.
@@ -41,12 +45,17 @@ func NewLoadCache(rdb redis.Cmdable) *LoadCache {
// The hash fields match the names in doc/03 §4: online_count, bw_up, bw_down, cpu.
func (c *LoadCache) Set(ctx context.Context, nodeUUID string, load NodeLoad) error {
key := nodeLoadKeyPrefix + nodeUUID
dpHealthy := 0
if load.DataPlaneHealthy {
dpHealthy = 1
}
pipe := c.rdb.Pipeline()
pipe.HSet(ctx, key,
"online_count", load.OnlineCount,
"bw_up", load.BandwidthUpBps,
"bw_down", load.BandwidthDownBps,
"cpu", load.CPUPercent,
"dp_healthy", dpHealthy,
)
pipe.Expire(ctx, key, nodeLoadTTL)
if _, err := pipe.Exec(ctx); err != nil {
@@ -66,7 +75,12 @@ func (c *LoadCache) Get(ctx context.Context, nodeUUID string) (*NodeLoad, bool,
if len(vals) == 0 {
return nil, false, nil
}
var l NodeLoad
// Default healthy: hashes written before dp_healthy existed (or by an older
// control plane during rollout) must not be read as "unhealthy" → false-down.
l := NodeLoad{DataPlaneHealthy: true}
if v, ok := vals["dp_healthy"]; ok {
l.DataPlaneHealthy = v == "1"
}
if v, ok := vals["online_count"]; ok {
_, _ = fmt.Sscan(v, &l.OnlineCount)
}