feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗
盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通) 而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。 控制面: - agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖), 经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。 - NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。 - effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode 统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。 客户端(connection_provider 连通看门狗): - 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。 - 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。 - 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。 测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗 智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -183,6 +183,7 @@ func (h *Handler) Heartbeat(ctx context.Context, req *agentv1.HeartbeatRequest)
|
||||
BandwidthUpBps: req.BandwidthUpBps,
|
||||
BandwidthDownBps: req.BandwidthDownBps,
|
||||
CPUPercent: req.CPUPercent,
|
||||
DataPlaneHealthy: req.DataPlaneHealthy,
|
||||
}); err != nil {
|
||||
slog.Warn("nodes.Handler.Heartbeat: load write failed",
|
||||
"node_uuid", nodeUUID, "err", err)
|
||||
|
||||
@@ -24,6 +24,10 @@ type NodeLoad struct {
|
||||
BandwidthUpBps int64
|
||||
BandwidthDownBps int64
|
||||
CPUPercent float64
|
||||
// DataPlaneHealthy is the agent's last sing-box health probe result. The
|
||||
// control plane downgrades a node to "down" when this is false (even if the
|
||||
// agent's gRPC stream is still up). Absent in older hashes → read as true.
|
||||
DataPlaneHealthy bool
|
||||
}
|
||||
|
||||
// LoadCache reads and writes node:load Redis hashes.
|
||||
@@ -41,12 +45,17 @@ func NewLoadCache(rdb redis.Cmdable) *LoadCache {
|
||||
// The hash fields match the names in doc/03 §4: online_count, bw_up, bw_down, cpu.
|
||||
func (c *LoadCache) Set(ctx context.Context, nodeUUID string, load NodeLoad) error {
|
||||
key := nodeLoadKeyPrefix + nodeUUID
|
||||
dpHealthy := 0
|
||||
if load.DataPlaneHealthy {
|
||||
dpHealthy = 1
|
||||
}
|
||||
pipe := c.rdb.Pipeline()
|
||||
pipe.HSet(ctx, key,
|
||||
"online_count", load.OnlineCount,
|
||||
"bw_up", load.BandwidthUpBps,
|
||||
"bw_down", load.BandwidthDownBps,
|
||||
"cpu", load.CPUPercent,
|
||||
"dp_healthy", dpHealthy,
|
||||
)
|
||||
pipe.Expire(ctx, key, nodeLoadTTL)
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
@@ -66,7 +75,12 @@ func (c *LoadCache) Get(ctx context.Context, nodeUUID string) (*NodeLoad, bool,
|
||||
if len(vals) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
var l NodeLoad
|
||||
// Default healthy: hashes written before dp_healthy existed (or by an older
|
||||
// control plane during rollout) must not be read as "unhealthy" → false-down.
|
||||
l := NodeLoad{DataPlaneHealthy: true}
|
||||
if v, ok := vals["dp_healthy"]; ok {
|
||||
l.DataPlaneHealthy = v == "1"
|
||||
}
|
||||
if v, ok := vals["online_count"]; ok {
|
||||
_, _ = fmt.Sscan(v, &l.OnlineCount)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user